From 6d6e3ddc44cb8245be6884a8bc28ee5c695e39dd Mon Sep 17 00:00:00 2001 From: bowenlan-amzn Date: Mon, 30 Jun 2025 08:53:50 -0700 Subject: [PATCH 01/51] Fix regex query to work with field alias (#18215) --------- Signed-off-by: bowenlan-amzn Signed-off-by: Ankit Jain Co-authored-by: Ankit Jain --- CHANGELOG.md | 2 +- .../search/61_query_string_field_alias.yml | 56 +++++++++++++++++++ .../index/search/QueryStringQueryParser.java | 9 ++- .../query/QueryStringQueryBuilderTests.java | 10 ++++ 4 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/61_query_string_field_alias.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 85b50967e9fb7..5496b0efd18bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,11 +33,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Add task cancellation checks in aggregators ([#18426](https://github.com/opensearch-project/OpenSearch/pull/18426)) - Fix concurrent timings in profiler ([#18540](https://github.com/opensearch-project/OpenSearch/pull/18540)) +- Fix regex query from query string query to work with field alias ([#18215](https://github.com/opensearch-project/OpenSearch/issues/18215)) - [Autotagging] Fix delete rule event consumption in InMemoryRuleProcessingService ([#18628](https://github.com/opensearch-project/OpenSearch/pull/18628)) - Cannot communicate with HTTP/2 when reactor-netty is enabled ([#18599](https://github.com/opensearch-project/OpenSearch/pull/18599)) - Fix the visit of sub queries for HasParentQuery and HasChildQuery ([#18621](https://github.com/opensearch-project/OpenSearch/pull/18621)) - ### Security [Unreleased 3.x]: https://github.com/opensearch-project/OpenSearch/compare/3.1...main diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/61_query_string_field_alias.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/61_query_string_field_alias.yml new file mode 100644 index 0000000000000..64926b1333174 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/61_query_string_field_alias.yml @@ -0,0 +1,56 @@ +setup: + - skip: + version: " - 3.1.99" + reason: "regex query over field alias support starts 3.2" + + - do: + indices.create: + index: test_index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + test: + type: text + test_alias: + type: alias + path: test + + - do: + bulk: + refresh: true + body: | + {"index":{"_index":"test_index","_id":"1"}} + {"test":"hello"} + {"index":{"_index":"test_index","_id":"2"}} + {"test":"world"} + +--- +"regex search on normal field": + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: + query: + query_string: + query: "test: /h[a-z].*/" + + - match: {hits.total: 1} + - match: {hits.hits.0._id: "1"} + +--- +"regex search on alias field": + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: + query: + query_string: + query: "test_alias: /h[a-z].*/" + + - match: {hits.total: 1} + - match: {hits.hits.0._id: "1"} diff --git a/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java b/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java index e67d9c7e08908..7bfb3475d6744 100644 --- a/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java +++ b/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java @@ -56,6 +56,7 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.RegExp; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.regex.Regex; import org.opensearch.common.unit.Fuzziness; @@ -787,8 +788,12 @@ private Query getRegexpQuerySingle(String field, String termStr) throws ParseExc if (currentFieldType == null) { return newUnmappedFieldQuery(field); } - setAnalyzer(getSearchAnalyzer(currentFieldType)); - return super.getRegexpQuery(field, termStr); + if (forceAnalyzer != null) { + setAnalyzer(forceAnalyzer); + } + // query string query normalizes search value + termStr = getAnalyzer().normalize(currentFieldType.name(), termStr).utf8ToString(); + return currentFieldType.regexpQuery(termStr, RegExp.ALL, 0, getDeterminizeWorkLimit(), getMultiTermRewriteMethod(), context); } catch (RuntimeException e) { if (lenient) { return newLenientFieldQuery(field, e); diff --git a/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java index f62bcbcf1038c..267567f0c3c86 100644 --- a/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java @@ -786,6 +786,16 @@ public void testToQueryRegExpQuery() throws Exception { assertTrue(regexpQuery.toString().contains("/foo*bar/")); } + public void testRegexpQueryParserWithForceAnalyzer() throws Exception { + QueryStringQueryParser queryParser = new QueryStringQueryParser(createShardContext(), TEXT_FIELD_NAME); + queryParser.setForceAnalyzer(new org.apache.lucene.analysis.standard.StandardAnalyzer()); + Query query = queryParser.parse("/aBc.*/"); + assertThat(query, instanceOf(RegexpQuery.class)); + RegexpQuery regexpQuery = (RegexpQuery) query; + // Standard analyzer normalizes to lowercase, verifying the normalization path with currentFieldType.name() is hit + assertTrue(regexpQuery.toString().contains("abc.*")); + } + public void testToQueryRegExpQueryTooComplex() throws Exception { QueryStringQueryBuilder queryBuilder = queryStringQuery("/[ac]*a[ac]{50,200}/").defaultField(TEXT_FIELD_NAME); From 8e0675f0866d28650bd84f31754889587360c1c4 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Mon, 30 Jun 2025 10:49:16 -0700 Subject: [PATCH 02/51] Add workflow to automatically trigger manifest generation process (#18629) Signed-off-by: Sayali Gaikawad --- .../workflows/trigger-manifest-generation.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/trigger-manifest-generation.yml diff --git a/.github/workflows/trigger-manifest-generation.yml b/.github/workflows/trigger-manifest-generation.yml new file mode 100644 index 0000000000000..3cd3b78181cf6 --- /dev/null +++ b/.github/workflows/trigger-manifest-generation.yml @@ -0,0 +1,17 @@ +name: Trigger manifest generation workflow + +on: + workflow_dispatch: + push: + paths: + - buildSrc/version.properties + +jobs: + trigger-manifest-workflow: + if: github.repository == 'opensearch-project/OpenSearch' + runs-on: ubuntu-latest + steps: + - name: Trigger manifest-update workflow + run: | + echo "Triggering manifest-update workflow at https://build.ci.opensearch.org/job/manifest-update/" + curl -f -X POST https://build.ci.opensearch.org/job/manifest-update/build --user "${{ secrets.JENKINS_GITHUB_USER}}:${{ secrets.JENKINS_GITHUB_USER_TOKEN}}" \ No newline at end of file From c230f4e1d9fbcf76857976693d1eb091f6579594 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jun 2025 14:13:34 -0400 Subject: [PATCH 03/51] Bump com.squareup.okio:okio from 3.13.0 to 3.14.0 in /test/fixtures/hdfs-fixture (#18645) * Bump com.squareup.okio:okio in /test/fixtures/hdfs-fixture Bumps [com.squareup.okio:okio](https://github.com/square/okio) from 3.13.0 to 3.14.0. - [Release notes](https://github.com/square/okio/releases) - [Changelog](https://github.com/square/okio/blob/master/CHANGELOG.md) - [Commits](https://github.com/square/okio/compare/parent-3.13.0...parent-3.14.0) --- updated-dependencies: - dependency-name: com.squareup.okio:okio dependency-version: 3.14.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5496b0efd18bb..0aa60faea6cab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.logging.log4j:log4j-core` from 2.24.3 to 2.25.0 ([#18589](https://github.com/opensearch-project/OpenSearch/pull/18589)) - Bump `com.google.code.gson:gson` from 2.13.0 to 2.13.1 ([#18585](https://github.com/opensearch-project/OpenSearch/pull/18585)) - Bump `com.azure:azure-core-http-netty` from 1.15.11 to 1.15.12 ([#18586](https://github.com/opensearch-project/OpenSearch/pull/18586)) +- Bump `com.squareup.okio:okio` from 3.13.0 to 3.14.0 ([#18645](https://github.com/opensearch-project/OpenSearch/pull/18645)) ### Deprecated diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 396cd6feceb90..6d6851e2eab50 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -88,6 +88,6 @@ dependencies { runtimeOnly("com.squareup.okhttp3:okhttp:4.12.0") { exclude group: "com.squareup.okio" } - runtimeOnly "com.squareup.okio:okio:3.13.0" + runtimeOnly "com.squareup.okio:okio:3.14.0" runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.7" } From 4e92a8000b202b28a9a27c51db055d92d87baba6 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Mon, 30 Jun 2025 13:10:25 -0700 Subject: [PATCH 04/51] Fix credentials format (#18651) Signed-off-by: Sayali Gaikawad --- .github/workflows/trigger-manifest-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trigger-manifest-generation.yml b/.github/workflows/trigger-manifest-generation.yml index 3cd3b78181cf6..ddee9c3710d3b 100644 --- a/.github/workflows/trigger-manifest-generation.yml +++ b/.github/workflows/trigger-manifest-generation.yml @@ -14,4 +14,4 @@ jobs: - name: Trigger manifest-update workflow run: | echo "Triggering manifest-update workflow at https://build.ci.opensearch.org/job/manifest-update/" - curl -f -X POST https://build.ci.opensearch.org/job/manifest-update/build --user "${{ secrets.JENKINS_GITHUB_USER}}:${{ secrets.JENKINS_GITHUB_USER_TOKEN}}" \ No newline at end of file + curl -f -X POST https://build.ci.opensearch.org/job/manifest-update/build --user ${{ secrets.JENKINS_GITHUB_USER}}:${{ secrets.JENKINS_GITHUB_USER_TOKEN}} \ No newline at end of file From d04e2f9a626626b1535da87283cd8dbfc037884c Mon Sep 17 00:00:00 2001 From: Finn Date: Mon, 30 Jun 2025 13:15:54 -0700 Subject: [PATCH 05/51] Provide aux transport to SecureAuxTransportSettingsProvider (#18616) * Pass aux transport to secure settings parameters(). Signed-off-by: Finn Carroll * Update javadocs, fix UTs. Signed-off-by: Finn Carroll * Spotless apply Signed-off-by: Finn Carroll * Pass aux transport key as unique identifier instead of transport ref. Signed-off-by: Finn Carroll * Consistent naming. auxTransportSettingKey -> auxTransportType. Signed-off-by: Finn Carroll * Changelog. Signed-off-by: Finn Carroll * Spotless apply Signed-off-by: Finn Carroll --------- Signed-off-by: Finn Carroll --- CHANGELOG.md | 1 + .../grpc/ssl/SecureNetty4GrpcServerTransport.java | 5 +++-- .../transport/grpc/ssl/SecureSettingsHelpers.java | 5 ++--- .../plugins/SecureAuxTransportSettingsProvider.java | 10 +++++++--- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0aa60faea6cab..926683b5fafd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Changed - Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570)) +- Update SecureAuxTransportSettingsProvider to distinguish between aux transport types ([#18616](https://github.com/opensearch-project/OpenSearch/pull/18616)) ### Dependencies - Bump `stefanzweifel/git-auto-commit-action` from 5 to 6 ([#18524](https://github.com/opensearch-project/OpenSearch/pull/18524)) diff --git a/plugins/transport-grpc/src/main/java/org/opensearch/plugin/transport/grpc/ssl/SecureNetty4GrpcServerTransport.java b/plugins/transport-grpc/src/main/java/org/opensearch/plugin/transport/grpc/ssl/SecureNetty4GrpcServerTransport.java index a886679ada293..3facd6305f176 100644 --- a/plugins/transport-grpc/src/main/java/org/opensearch/plugin/transport/grpc/ssl/SecureNetty4GrpcServerTransport.java +++ b/plugins/transport-grpc/src/main/java/org/opensearch/plugin/transport/grpc/ssl/SecureNetty4GrpcServerTransport.java @@ -106,7 +106,7 @@ public String settingKey() { * @param provider for SSLContext and SecureAuxTransportParameters (ClientAuth and enabled ciphers). */ private JdkSslContext getSslContext(Settings settings, SecureAuxTransportSettingsProvider provider) throws SSLException { - Optional sslContext = provider.buildSecureAuxServerTransportContext(settings, this); + Optional sslContext = provider.buildSecureAuxServerTransportContext(settings, this.settingKey()); if (sslContext.isEmpty()) { try { sslContext = Optional.of(SSLContext.getDefault()); @@ -114,7 +114,8 @@ private JdkSslContext getSslContext(Settings settings, SecureAuxTransportSetting throw new SSLException("Failed to build default SSLContext for " + SecureNetty4GrpcServerTransport.class.getName(), e); } } - SecureAuxTransportSettingsProvider.SecureAuxTransportParameters params = provider.parameters().orElseGet(DefaultParameters::new); + SecureAuxTransportSettingsProvider.SecureAuxTransportParameters params = provider.parameters(settings, this.settingKey()) + .orElseGet(DefaultParameters::new); ClientAuth clientAuth = ClientAuth.valueOf(params.clientAuth().orElseThrow().toUpperCase(Locale.ROOT)); return new JdkSslContext( sslContext.get(), diff --git a/plugins/transport-grpc/src/test/java/org/opensearch/plugin/transport/grpc/ssl/SecureSettingsHelpers.java b/plugins/transport-grpc/src/test/java/org/opensearch/plugin/transport/grpc/ssl/SecureSettingsHelpers.java index 387889eb87ae0..5cc65ee615a2a 100644 --- a/plugins/transport-grpc/src/test/java/org/opensearch/plugin/transport/grpc/ssl/SecureSettingsHelpers.java +++ b/plugins/transport-grpc/src/test/java/org/opensearch/plugin/transport/grpc/ssl/SecureSettingsHelpers.java @@ -10,7 +10,6 @@ import org.opensearch.common.settings.Settings; import org.opensearch.plugins.SecureAuxTransportSettingsProvider; -import org.opensearch.transport.AuxTransport; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; @@ -110,7 +109,7 @@ static SecureAuxTransportSettingsProvider getSecureSettingsProvider( ) { return new SecureAuxTransportSettingsProvider() { @Override - public Optional buildSecureAuxServerTransportContext(Settings settings, AuxTransport transport) + public Optional buildSecureAuxServerTransportContext(Settings settings, String auxTransportType) throws SSLException { // Choose a random protocol from among supported test defaults String protocol = randomFrom(DEFAULT_SSL_PROTOCOLS); @@ -126,7 +125,7 @@ public Optional buildSecureAuxServerTransportContext(Settings settin } @Override - public Optional parameters() { + public Optional parameters(Settings settings, String auxTransportType) { return Optional.of(new SecureAuxTransportParameters() { @Override public Optional clientAuth() { diff --git a/server/src/main/java/org/opensearch/plugins/SecureAuxTransportSettingsProvider.java b/server/src/main/java/org/opensearch/plugins/SecureAuxTransportSettingsProvider.java index f90d642409b01..826d5ca641b22 100644 --- a/server/src/main/java/org/opensearch/plugins/SecureAuxTransportSettingsProvider.java +++ b/server/src/main/java/org/opensearch/plugins/SecureAuxTransportSettingsProvider.java @@ -10,7 +10,6 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.settings.Settings; -import org.opensearch.transport.AuxTransport; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLException; @@ -26,17 +25,22 @@ public interface SecureAuxTransportSettingsProvider { /** * Fetch an SSLContext as managed by pluggable security provider. + * @param settings for providing additional configuration options when building the ssl context. + * @param auxTransportType key for enabling this transport with AUX_TRANSPORT_TYPES_SETTING. * @return an instance of SSLContext. */ - default Optional buildSecureAuxServerTransportContext(Settings settings, AuxTransport transport) throws SSLException { + default Optional buildSecureAuxServerTransportContext(Settings settings, String auxTransportType) throws SSLException { return Optional.empty(); } /** * Additional params required for configuring ALPN. + * @param settings for providing additional configuration options when building secure params. + * @param auxTransportType key for enabling this transport with AUX_TRANSPORT_TYPES_SETTING. * @return an instance of {@link SecureAuxTransportSettingsProvider.SecureAuxTransportParameters} */ - default Optional parameters() { + default Optional parameters(Settings settings, String auxTransportType) + throws SSLException { return Optional.empty(); } From d990978e153026d3ce7ef9d3a9eabd405f7d52d2 Mon Sep 17 00:00:00 2001 From: kkewwei Date: Tue, 1 Jul 2025 14:27:49 +0800 Subject: [PATCH 06/51] Making snapshot store/restore rate dynamic (#18069) * Making snapshot store/restore rate dynamic Signed-off-by: kkewwei Signed-off-by: kkewwei * simple the code Signed-off-by: kkewwei Signed-off-by: kkewwei --------- Signed-off-by: kkewwei Signed-off-by: kkewwei --- CHANGELOG.md | 1 + .../repositories/hdfs/HdfsTests.java | 6 +- .../repositories/RepositoriesServiceIT.java | 133 +++++++++++++++++- .../snapshots/ConcurrentSnapshotsIT.java | 9 +- .../repositories/RepositoriesService.java | 49 +++++-- .../opensearch/repositories/Repository.java | 4 + .../blobstore/BlobStoreRepository.java | 125 +++++++++++++--- 7 files changed, 286 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 926683b5fafd0..231aa8c254b54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Changed - Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570)) - Update SecureAuxTransportSettingsProvider to distinguish between aux transport types ([#18616](https://github.com/opensearch-project/OpenSearch/pull/18616)) +- Making multi rate limiters in repository dynamic [#18069](https://github.com/opensearch-project/OpenSearch/pull/18069) ### Dependencies - Bump `stefanzweifel/git-auto-commit-action` from 5 to 6 ([#18524](https://github.com/opensearch-project/OpenSearch/pull/18524)) diff --git a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsTests.java b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsTests.java index 297a3052cc6d9..aabeb34b80ae5 100644 --- a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsTests.java +++ b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsTests.java @@ -160,7 +160,7 @@ public void testSimpleWorkflow() { public void testMissingUri() { try { - OpenSearchIntegTestCase.putRepository(client().admin().cluster(), "test-repo", "hdfs", Settings.builder()); + OpenSearchIntegTestCase.putRepository(client().admin().cluster(), "test-repo1", "hdfs", Settings.builder()); fail(); } catch (RepositoryException e) { assertTrue(e.getCause() instanceof IllegalArgumentException); @@ -193,7 +193,7 @@ public void testNonHdfsUri() { public void testPathSpecifiedInHdfs() { try { Settings.Builder settings = Settings.builder().put("uri", "hdfs:///some/path"); - OpenSearchIntegTestCase.putRepository(client().admin().cluster(), "test-repo", "hdfs", settings); + OpenSearchIntegTestCase.putRepository(client().admin().cluster(), "test-repo2", "hdfs", settings); fail(); } catch (RepositoryException e) { assertTrue(e.getCause() instanceof IllegalArgumentException); @@ -204,7 +204,7 @@ public void testPathSpecifiedInHdfs() { public void testMissingPath() { try { Settings.Builder settings = Settings.builder().put("uri", "hdfs:///"); - OpenSearchIntegTestCase.putRepository(client().admin().cluster(), "test-repo", "hdfs", settings); + OpenSearchIntegTestCase.putRepository(client().admin().cluster(), "test-repo3", "hdfs", settings); fail(); } catch (RepositoryException e) { assertTrue(e.getCause() instanceof IllegalArgumentException); diff --git a/server/src/internalClusterTest/java/org/opensearch/repositories/RepositoriesServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/repositories/RepositoriesServiceIT.java index b5d5bddd160ca..5f86fd791821e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/repositories/RepositoriesServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/repositories/RepositoriesServiceIT.java @@ -32,6 +32,7 @@ package org.opensearch.repositories; +import org.apache.lucene.store.RateLimiter; import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.common.settings.Settings; @@ -42,12 +43,18 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.transport.client.Client; +import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.concurrent.atomic.AtomicInteger; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.MAX_REMOTE_DOWNLOAD_BYTES_PER_SEC; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.MAX_REMOTE_LOW_PRIORITY_UPLOAD_BYTES_PER_SEC; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.MAX_REMOTE_UPLOAD_BYTES_PER_SEC; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.MAX_RESTORE_BYTES_PER_SEC; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.MAX_SNAPSHOT_BYTES_PER_SEC; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -138,10 +145,11 @@ public void testCreatSnapAndUpdateReposityCauseInfiniteLoop() throws Interrupted // create repository final String repositoryName = "test-repo"; + Path path = randomRepoPath(); Settings.Builder repoSettings = Settings.builder() - .put("location", randomRepoPath()) - .put("max_snapshot_bytes_per_sec", "10mb") - .put("max_restore_bytes_per_sec", "10mb"); + .put("location", path) + .put(MAX_SNAPSHOT_BYTES_PER_SEC, "10mb") + .put(MAX_RESTORE_BYTES_PER_SEC, "10mb"); OpenSearchIntegTestCase.putRepositoryWithNoSettingOverrides( client().admin().cluster(), repositoryName, @@ -176,7 +184,7 @@ public void testCreatSnapAndUpdateReposityCauseInfiniteLoop() throws Interrupted try { logger.info("--> begin to reset repository"); - repoSettings = Settings.builder().put("location", randomRepoPath()).put("max_snapshot_bytes_per_sec", "300mb"); + repoSettings = Settings.builder().put("location", randomRepoPath()).put(MAX_SNAPSHOT_BYTES_PER_SEC, "300mb"); OpenSearchIntegTestCase.putRepositoryWithNoSettingOverrides( client().admin().cluster(), repositoryName, @@ -194,4 +202,121 @@ public void testCreatSnapAndUpdateReposityCauseInfiniteLoop() throws Interrupted thread.join(); } + + public void testAdjustBytesPerSecSettingForSnapAndRestore() { + final InternalTestCluster cluster = internalCluster(); + final RepositoriesService repositoriesService = cluster.getDataOrClusterManagerNodeInstances(RepositoriesService.class) + .iterator() + .next(); + + // create repository + final String repositoryName = "test-repo1"; + long rateBytes = 200000; + Path path = randomRepoPath(); + Settings.Builder repoSettings = Settings.builder() + .put("location", path) + .put(MAX_SNAPSHOT_BYTES_PER_SEC, (rateBytes + "b")) + .put(MAX_RESTORE_BYTES_PER_SEC, (rateBytes + "b")) + .put(MAX_REMOTE_UPLOAD_BYTES_PER_SEC, (rateBytes + "b")) + .put(MAX_REMOTE_LOW_PRIORITY_UPLOAD_BYTES_PER_SEC, (rateBytes + "b")) + .put(MAX_REMOTE_DOWNLOAD_BYTES_PER_SEC, (rateBytes + "b")); + OpenSearchIntegTestCase.putRepositoryWithNoSettingOverrides( + client().admin().cluster(), + repositoryName, + FsRepository.TYPE, + true, + repoSettings + ); + + FsRepository repository = (FsRepository) repositoriesService.repository(repositoryName); + RateLimiter snapshotRateLimiter = repository.snapshotRateLimiter(); + assertThat(snapshotRateLimiter.getMBPerSec(), equalTo((double) rateBytes / (1024 * 1024))); + RateLimiter restoreRateLimiter = repository.restoreRateLimiter(); + assertThat(restoreRateLimiter.getMBPerSec(), equalTo((double) rateBytes / (1024 * 1024))); + RateLimiter remoteUploadRateLimiter = repository.remoteUploadRateLimiter(); + assertThat(remoteUploadRateLimiter.getMBPerSec(), equalTo((double) rateBytes / (1024 * 1024))); + RateLimiter remoteUploadLowPriorityRateLimiter = repository.remoteUploadLowPriorityRateLimiter(); + assertThat(remoteUploadLowPriorityRateLimiter.getMBPerSec(), equalTo((double) rateBytes / (1024 * 1024))); + RateLimiter remoteDownloadRateLimiter = repository.remoteDownloadRateLimiter(); + assertThat(remoteDownloadRateLimiter.getMBPerSec(), equalTo((double) rateBytes / (1024 * 1024))); + + // adjust all the reloadable settings + { + rateBytes = rateBytes / 2; + repoSettings = Settings.builder() + .put(MAX_SNAPSHOT_BYTES_PER_SEC, (rateBytes + "b")) + .put(MAX_RESTORE_BYTES_PER_SEC, (rateBytes + "b")) + .put(MAX_REMOTE_UPLOAD_BYTES_PER_SEC, (rateBytes + "b")) + .put(MAX_REMOTE_LOW_PRIORITY_UPLOAD_BYTES_PER_SEC, (rateBytes + "b")) + .put(MAX_REMOTE_DOWNLOAD_BYTES_PER_SEC, (rateBytes + "b")); + OpenSearchIntegTestCase.putRepositoryWithNoSettingOverrides( + client().admin().cluster(), + repositoryName, + FsRepository.TYPE, + true, + repoSettings + ); + FsRepository newRepository = (FsRepository) repositoriesService.repository(repositoryName); + assertThat(newRepository, sameInstance(repository)); + snapshotRateLimiter = newRepository.snapshotRateLimiter(); + assertThat(snapshotRateLimiter.getMBPerSec(), equalTo((double) rateBytes / (1024 * 1024))); + restoreRateLimiter = newRepository.restoreRateLimiter(); + assertThat(restoreRateLimiter.getMBPerSec(), equalTo((double) rateBytes / (1024 * 1024))); + remoteUploadRateLimiter = newRepository.remoteUploadRateLimiter(); + assertThat(remoteUploadRateLimiter.getMBPerSec(), equalTo((double) rateBytes / (1024 * 1024))); + remoteUploadLowPriorityRateLimiter = newRepository.remoteUploadLowPriorityRateLimiter(); + assertThat(remoteUploadLowPriorityRateLimiter.getMBPerSec(), equalTo((double) rateBytes / (1024 * 1024))); + remoteDownloadRateLimiter = newRepository.remoteDownloadRateLimiter(); + assertThat(remoteDownloadRateLimiter.getMBPerSec(), equalTo((double) rateBytes / (1024 * 1024))); + } + + // In addition to the settings in RELOADABLE_SETTINGS, all the new settings should be equal to current settings + { + long newRateBytes = rateBytes / 2; + repoSettings = Settings.builder() + .put("location", path) + .put(MAX_SNAPSHOT_BYTES_PER_SEC, (newRateBytes + "b")) + .put(MAX_RESTORE_BYTES_PER_SEC, (newRateBytes + "b")); + OpenSearchIntegTestCase.putRepositoryWithNoSettingOverrides( + client().admin().cluster(), + repositoryName, + FsRepository.TYPE, + true, + repoSettings + ); + FsRepository newRepository = (FsRepository) repositoriesService.repository(repositoryName); + assertThat(newRepository, sameInstance(repository)); + snapshotRateLimiter = newRepository.snapshotRateLimiter(); + assertThat(snapshotRateLimiter.getMBPerSec(), equalTo((double) newRateBytes / (1024 * 1024))); + restoreRateLimiter = newRepository.restoreRateLimiter(); + assertThat(restoreRateLimiter.getMBPerSec(), equalTo((double) newRateBytes / (1024 * 1024))); + remoteUploadRateLimiter = newRepository.remoteUploadRateLimiter(); + assertThat(remoteUploadRateLimiter.getMBPerSec(), equalTo((double) rateBytes / (1024 * 1024))); + remoteUploadLowPriorityRateLimiter = newRepository.remoteUploadLowPriorityRateLimiter(); + assertThat(remoteUploadLowPriorityRateLimiter.getMBPerSec(), equalTo((double) rateBytes / (1024 * 1024))); + remoteDownloadRateLimiter = newRepository.remoteDownloadRateLimiter(); + assertThat(remoteDownloadRateLimiter.getMBPerSec(), equalTo((double) rateBytes / (1024 * 1024))); + } + + // the new settings are not all equal to the old settings, so the repository will be not reloaded + { + rateBytes = rateBytes / 2; + repoSettings = Settings.builder() + .put("location", path) + .put("io_buffer_size", "8mb") + .put(MAX_RESTORE_BYTES_PER_SEC, (rateBytes + "b")) + .put(MAX_REMOTE_UPLOAD_BYTES_PER_SEC, (rateBytes + "b")) + .put(MAX_REMOTE_LOW_PRIORITY_UPLOAD_BYTES_PER_SEC, (rateBytes + "b")) + .put(MAX_REMOTE_DOWNLOAD_BYTES_PER_SEC, (rateBytes + "b")); + OpenSearchIntegTestCase.putRepositoryWithNoSettingOverrides( + client().admin().cluster(), + repositoryName, + FsRepository.TYPE, + true, + repoSettings + ); + FsRepository newRepository = (FsRepository) repositoriesService.repository(repositoryName); + assertNotEquals(newRepository, repository); + } + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java index 252efcdc979bb..af17ef596b6a4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java @@ -50,6 +50,7 @@ import org.opensearch.common.util.concurrent.UncategorizedExecutionException; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.rest.RestStatus; import org.opensearch.discovery.AbstractDisruptionTestCase; import org.opensearch.plugins.Plugin; @@ -154,6 +155,7 @@ public void testSettingsUpdateFailWhenCreateSnapshotInProgress() throws Exceptio Thread.sleep(1000); // Wait for the snapshot to start assertFalse(createSlowFuture.isDone()); // Ensure the snapshot is still in progress // Attempt to update the repository settings while the snapshot is in progress + settings.put("chunk_size", 2000, ByteSizeUnit.BYTES); IllegalStateException ex = assertThrows(IllegalStateException.class, () -> updateRepository(repoName, "mock", settings)); // Verify that the update fails with an appropriate exception assertEquals("trying to modify or unregister repository that is currently used", ex.getMessage()); @@ -180,10 +182,9 @@ public void testSettingsUpdateFailWhenDeleteSnapshotInProgress() throws Interrup Thread.sleep(1000); // Wait for the delete operation to start assertFalse(future.isDone()); // Ensure the delete operation is still in progress // Attempt to update the repository settings while the delete operation is in progress - IllegalStateException ex = assertThrows( - IllegalStateException.class, - () -> updateRepository(repoName, "mock", randomRepositorySettings()) - ); + Settings.Builder newSettings = randomRepositorySettings(); + newSettings.put("chunk_size", 2000, ByteSizeUnit.BYTES); + IllegalStateException ex = assertThrows(IllegalStateException.class, () -> updateRepository(repoName, "mock", newSettings)); // Verify that the update fails with an appropriate exception assertEquals("trying to modify or unregister repository that is currently used", ex.getMessage()); unblockNode(repoName, clusterManagerName); // Unblock the delete operation diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index 994c981b8bb0c..1060c946ae799 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -174,7 +174,7 @@ public RepositoriesService( public void registerOrUpdateRepository(final PutRepositoryRequest request, final ActionListener listener) { assert lifecycle.started() : "Trying to register new repository but service is in state [" + lifecycle.state() + "]"; - final RepositoryMetadata newRepositoryMetadata = new RepositoryMetadata( + RepositoryMetadata newRepositoryMetadata = new RepositoryMetadata( request.name(), request.type(), request.settings(), @@ -206,14 +206,32 @@ public void registerOrUpdateRepository(final PutRepositoryRequest request, final registrationListener = listener; } - // Trying to create the new repository on cluster-manager to make sure it works - try { - closeRepository(createRepository(newRepositoryMetadata, typesRegistry)); - } catch (Exception e) { - registrationListener.onFailure(e); - return; + Repository currentRepository = repositories.get(request.name()); + boolean isReloadableSettings = currentRepository != null && currentRepository.isReloadableSettings(newRepositoryMetadata); + + if (isReloadableSettings) { + // We are reloading the repository, so we need to preserve the old settings in the new repository metadata + Settings updatedSettings = Settings.builder() + .put(currentRepository.getMetadata().settings()) + .put(newRepositoryMetadata.settings()) + .build(); + newRepositoryMetadata = new RepositoryMetadata( + newRepositoryMetadata.name(), + newRepositoryMetadata.type(), + updatedSettings, + newRepositoryMetadata.cryptoMetadata() + ); + } else { + // Trying to create the new repository on cluster-manager to make sure it works + try { + closeRepository(createRepository(newRepositoryMetadata, typesRegistry)); + } catch (Exception e) { + registrationListener.onFailure(e); + return; + } } + final RepositoryMetadata finalRepositoryMetadata = newRepositoryMetadata; clusterService.submitStateUpdateTask( "put_repository [" + request.name() + "]", new AckedClusterStateUpdateTask(request, registrationListener) { @@ -224,7 +242,9 @@ protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { @Override public ClusterState execute(ClusterState currentState) { - ensureRepositoryNotInUse(currentState, request.name()); + if (isReloadableSettings == false) { + ensureRepositoryNotInUse(currentState, request.name()); + } Metadata metadata = currentState.metadata(); Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata()); RepositoriesMetadata repositories = metadata.custom(RepositoriesMetadata.TYPE); @@ -245,17 +265,17 @@ public ClusterState execute(ClusterState currentState) { List repositoriesMetadata = new ArrayList<>(repositories.repositories().size() + 1); for (RepositoryMetadata repositoryMetadata : repositories.repositories()) { - RepositoryMetadata updatedRepositoryMetadata = newRepositoryMetadata; + RepositoryMetadata updatedRepositoryMetadata = finalRepositoryMetadata; if (isSystemRepositorySettingPresent(repositoryMetadata.settings())) { Settings updatedSettings = Settings.builder() - .put(newRepositoryMetadata.settings()) + .put(finalRepositoryMetadata.settings()) .put(SYSTEM_REPOSITORY_SETTING.getKey(), true) .build(); updatedRepositoryMetadata = new RepositoryMetadata( - newRepositoryMetadata.name(), - newRepositoryMetadata.type(), + finalRepositoryMetadata.name(), + finalRepositoryMetadata.type(), updatedSettings, - newRepositoryMetadata.cryptoMetadata() + finalRepositoryMetadata.cryptoMetadata() ); } if (repositoryMetadata.name().equals(updatedRepositoryMetadata.name())) { @@ -481,7 +501,8 @@ public void applyClusterState(ClusterChangedEvent event) { if (previousMetadata.type().equals(repositoryMetadata.type()) == false || previousMetadata.settings().equals(repositoryMetadata.settings()) == false) { // Previous version is different from the version in settings - if (repository.isSystemRepository() && repository.isReloadable()) { + if ((repository.isSystemRepository() && repository.isReloadable()) + || repository.isReloadableSettings(repositoryMetadata)) { logger.debug( "updating repository [{}] in-place to use new metadata [{}]", repositoryMetadata.name(), diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index 259c4a6e09ce7..521187f48b375 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -602,6 +602,10 @@ default boolean isReloadable() { return false; } + default boolean isReloadableSettings(RepositoryMetadata newRepositoryMetadata) { + return false; + } + /** * Reload the repository inplace */ diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 940942b816536..7cdbc31563654 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -264,6 +264,46 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp public static final long CACHE_DEFAULT_THRESHOLD = calculateDefaultSnapshotRepositoryDataCacheThreshold(); + public static final String MAX_SNAPSHOT_BYTES_PER_SEC = "max_snapshot_bytes_per_sec"; + + public static final Setting SNAPSHOT_BYTES_PER_SEC_SETTING = Setting.byteSizeSetting( + MAX_SNAPSHOT_BYTES_PER_SEC, + new ByteSizeValue(40, ByteSizeUnit.MB), + Setting.Property.NodeScope + ); + + public static final String MAX_RESTORE_BYTES_PER_SEC = "max_restore_bytes_per_sec"; + + public static final Setting RESTORE_BYTES_PER_SEC_SETTING = Setting.byteSizeSetting( + MAX_RESTORE_BYTES_PER_SEC, + ByteSizeValue.ZERO, + Setting.Property.NodeScope + ); + + public static final String MAX_REMOTE_UPLOAD_BYTES_PER_SEC = "max_remote_upload_bytes_per_sec"; + + public static final Setting MAX_REMOTE_UPLOAD_BYTES_PER_SEC_SETTING = Setting.byteSizeSetting( + MAX_REMOTE_UPLOAD_BYTES_PER_SEC, + ByteSizeValue.ZERO, + Setting.Property.NodeScope + ); + + public static final String MAX_REMOTE_LOW_PRIORITY_UPLOAD_BYTES_PER_SEC = "max_remote_low_priority_upload_bytes_per_sec"; + + public static final Setting MAX_REMOTE_LOW_PRIORITY_UPLOAD_BYTES_PER_SEC_SETTING = Setting.byteSizeSetting( + MAX_REMOTE_LOW_PRIORITY_UPLOAD_BYTES_PER_SEC, + ByteSizeValue.ZERO, + Setting.Property.NodeScope + ); + + public static final String MAX_REMOTE_DOWNLOAD_BYTES_PER_SEC = "max_remote_download_bytes_per_sec"; + + public static final Setting MAX_REMOTE_DOWNLOAD_BYTES_PER_SEC_SETTING = Setting.byteSizeSetting( + MAX_REMOTE_DOWNLOAD_BYTES_PER_SEC, + ByteSizeValue.ZERO, + Setting.Property.NodeScope + ); + /** * Set to Integer.MAX_VALUE - 8 to prevent OutOfMemoryError due to array header requirements, following the limit used in certain JDK versions. * This ensures compatibility across various JDK versions. For a practical usage example, @@ -328,6 +368,14 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp Setting.Property.NodeScope ); + private static final Set RELOADABLE_SETTINGS = Set.of( + MAX_RESTORE_BYTES_PER_SEC, + MAX_SNAPSHOT_BYTES_PER_SEC, + MAX_REMOTE_UPLOAD_BYTES_PER_SEC, + MAX_REMOTE_LOW_PRIORITY_UPLOAD_BYTES_PER_SEC, + MAX_REMOTE_DOWNLOAD_BYTES_PER_SEC + ); + public static long calculateDefaultSnapshotRepositoryDataCacheThreshold() { return Math.max(ByteSizeUnit.KB.toBytes(500), CACHE_MAX_THRESHOLD / 2); } @@ -592,15 +640,11 @@ private void readRepositoryMetadata(RepositoryMetadata repositoryMetadata) { this.metadata = repositoryMetadata; supportURLRepo = SUPPORT_URL_REPO.get(metadata.settings()); - snapshotRateLimiter = getRateLimiter(metadata.settings(), "max_snapshot_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB)); - restoreRateLimiter = getRateLimiter(metadata.settings(), "max_restore_bytes_per_sec", ByteSizeValue.ZERO); - remoteUploadRateLimiter = getRateLimiter(metadata.settings(), "max_remote_upload_bytes_per_sec", ByteSizeValue.ZERO); - remoteUploadLowPriorityRateLimiter = getRateLimiter( - metadata.settings(), - "max_remote_low_priority_upload_bytes_per_sec", - ByteSizeValue.ZERO - ); - remoteDownloadRateLimiter = getRateLimiter(metadata.settings(), "max_remote_download_bytes_per_sec", ByteSizeValue.ZERO); + snapshotRateLimiter = getRateLimiter(SNAPSHOT_BYTES_PER_SEC_SETTING, metadata.settings()); + restoreRateLimiter = getRateLimiter(RESTORE_BYTES_PER_SEC_SETTING, metadata.settings()); + remoteUploadRateLimiter = getRateLimiter(MAX_REMOTE_UPLOAD_BYTES_PER_SEC_SETTING, metadata.settings()); + remoteUploadLowPriorityRateLimiter = getRateLimiter(MAX_REMOTE_LOW_PRIORITY_UPLOAD_BYTES_PER_SEC_SETTING, metadata.settings()); + remoteDownloadRateLimiter = getRateLimiter(MAX_REMOTE_DOWNLOAD_BYTES_PER_SEC_SETTING, metadata.settings()); readOnly = READONLY_SETTING.get(metadata.settings()); cacheRepositoryData = CACHE_REPOSITORY_DATA.get(metadata.settings()); bufferSize = Math.toIntExact(BUFFER_SIZE_SETTING.get(metadata.settings()).getBytes()); @@ -2891,17 +2935,16 @@ private BlobPath shardPath(IndexId indexId, int shardId) { /** * Configures RateLimiter based on repository and global settings * - * @param repositorySettings repository settings - * @param setting setting to use to configure rate limiter - * @param defaultRate default limiting rate + * @param bytesPerSecSetting setting to use to configure rate limiter + * @param settings repository settings * @return rate limiter or null of no throttling is needed */ - private RateLimiter getRateLimiter(Settings repositorySettings, String setting, ByteSizeValue defaultRate) { - ByteSizeValue maxSnapshotBytesPerSec = repositorySettings.getAsBytesSize(setting, defaultRate); - if (maxSnapshotBytesPerSec.getBytes() <= 0) { + private RateLimiter getRateLimiter(Setting bytesPerSecSetting, Settings settings) { + ByteSizeValue maxByteSize = bytesPerSecSetting.get(settings); + if (maxByteSize.getBytes() <= 0) { return null; } else { - return new RateLimiter.SimpleRateLimiter(maxSnapshotBytesPerSec.getMbFrac()); + return new RateLimiter.SimpleRateLimiter(maxByteSize.getMbFrac()); } } @@ -4326,6 +4369,31 @@ public InputStream maybeRateLimitSnapshots(InputStream stream) { return maybeRateLimit(stream, () -> snapshotRateLimiter, snapshotRateLimitingTimeInNanos, BlobStoreTransferContext.SNAPSHOT); } + // Visible for testing + public RateLimiter snapshotRateLimiter() { + return snapshotRateLimiter; + } + + // Visible for testing + public RateLimiter restoreRateLimiter() { + return restoreRateLimiter; + } + + // Visible for testing + public RateLimiter remoteUploadRateLimiter() { + return remoteUploadRateLimiter; + } + + // Visible for testing + public RateLimiter remoteUploadLowPriorityRateLimiter() { + return remoteUploadLowPriorityRateLimiter; + } + + // Visible for testing + public RateLimiter remoteDownloadRateLimiter() { + return remoteDownloadRateLimiter; + } + @Override public List> getRestrictedSystemRepositorySettings() { return Arrays.asList(SYSTEM_REPOSITORY_SETTING, READONLY_SETTING, REMOTE_STORE_INDEX_SHALLOW_COPY); @@ -4715,6 +4783,31 @@ private static Optional extractShallowSnapshotUUID(String blobName) { return Optional.empty(); } + @Override + public boolean isReloadableSettings(RepositoryMetadata newRepositoryMetadata) { + if (metadata.name().equals(newRepositoryMetadata.name()) == false + || metadata.type().equals(newRepositoryMetadata.type()) == false + || Objects.equals(metadata.cryptoMetadata(), newRepositoryMetadata.cryptoMetadata()) == false) { + return false; + } + Settings newSettings = newRepositoryMetadata.settings(); + if (RELOADABLE_SETTINGS.containsAll(newSettings.keySet())) { + // the new settings are all contained in RELOADABLE_SETTINGS + return true; + } else { + Settings currentSettings = metadata.settings(); + // In addition to the settings in RELOADABLE_SETTINGS, all the new settings should be equal to current settings + Set allKeys = Stream.concat(newSettings.keySet().stream(), currentSettings.keySet().stream()) + .filter(key -> !RELOADABLE_SETTINGS.contains(key)) + .collect(Collectors.toSet()); + return allKeys.stream().allMatch(key -> isSettingEqual(newSettings, currentSettings, key)); + } + } + + private boolean isSettingEqual(Settings s1, Settings s2, String key) { + return Objects.equals(s1.get(key), s2.get(key)); + } + /** * The result of removing a snapshot from a shard folder in the repository. */ From 2cec4b55b5ff9d058eff279058930c135873d126 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 06:42:12 -0400 Subject: [PATCH 07/51] Bump com.netflix.nebula.ospackage-base from 11.11.2 to 12.0.0 in /distribution/packages (#18646) * Bump com.netflix.nebula.ospackage-base in /distribution/packages Bumps com.netflix.nebula.ospackage-base from 11.11.2 to 12.0.0. --- updated-dependencies: - dependency-name: com.netflix.nebula.ospackage-base dependency-version: 12.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: Craig Perkins Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Craig Perkins --- CHANGELOG.md | 1 + distribution/packages/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 231aa8c254b54..a4337a26966cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.google.code.gson:gson` from 2.13.0 to 2.13.1 ([#18585](https://github.com/opensearch-project/OpenSearch/pull/18585)) - Bump `com.azure:azure-core-http-netty` from 1.15.11 to 1.15.12 ([#18586](https://github.com/opensearch-project/OpenSearch/pull/18586)) - Bump `com.squareup.okio:okio` from 3.13.0 to 3.14.0 ([#18645](https://github.com/opensearch-project/OpenSearch/pull/18645)) +- Bump `com.netflix.nebula.ospackage-base` from 11.11.2 to 12.0.0 ([#18646](https://github.com/opensearch-project/OpenSearch/pull/18646)) ### Deprecated diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index a02f907f86aef..194c683da5ec7 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -63,7 +63,7 @@ import java.util.regex.Pattern */ plugins { - id "com.netflix.nebula.ospackage-base" version "11.11.2" + id "com.netflix.nebula.ospackage-base" version "12.0.0" } void addProcessFilesTask(String type, boolean jdk) { From 8fd9aebb0ca329c88de1cf74e6a037d5247a3593 Mon Sep 17 00:00:00 2001 From: TJ Neuenfeldt <56700214+neuenfeldttj@users.noreply.github.com> Date: Tue, 1 Jul 2025 06:04:59 -0700 Subject: [PATCH 08/51] Extend Profiler to allow for non-timing info (#18631) * copied from other branch, ready for pr Signed-off-by: TJ Neuenfeldt * cleanup from merging Signed-off-by: TJ Neuenfeldt * another spotless apply and fixed breaking change Signed-off-by: TJ Neuenfeldt * another spotlessApply Signed-off-by: TJ Neuenfeldt * forgot to change back concurrent agg stuff Signed-off-by: TJ Neuenfeldt * forgot to spotless after prev fix Signed-off-by: TJ Neuenfeldt * added more unit tests for coverage Signed-off-by: TJ Neuenfeldt * changed to supplier, removed timer ctors Signed-off-by: TJ Neuenfeldt * made feedback changes Signed-off-by: TJ Neuenfeldt * forgot javadoc for new class Signed-off-by: TJ Neuenfeldt * forgot to revert name to toString Signed-off-by: TJ Neuenfeldt * fix count check in concurrentquerypb Signed-off-by: TJ Neuenfeldt * unmodifiable map add Signed-off-by: TJ Neuenfeldt --------- Signed-off-by: TJ Neuenfeldt Signed-off-by: TJ Neuenfeldt <56700214+neuenfeldttj@users.noreply.github.com> --- CHANGELOG.md | 1 + .../search/internal/ContextIndexSearcher.java | 2 +- .../profile/AbstractInternalProfileTree.java | 2 +- .../profile/AbstractProfileBreakdown.java | 60 +++--- .../search/profile/AbstractProfiler.java | 2 +- .../profile/ContextualProfileBreakdown.java | 11 +- .../search/profile/ProfileMetric.java | 40 ++++ .../search/profile/ProfileMetricUtil.java | 38 ++++ .../search/profile/ProfileResult.java | 2 +- .../opensearch/search/profile/Profilers.java | 2 +- .../org/opensearch/search/profile/Timer.java | 23 ++- .../AggregationProfileBreakdown.java | 13 +- .../ConcurrentAggregationProfiler.java | 9 +- .../query/AbstractQueryProfileTree.java | 4 +- .../ConcurrentQueryProfileBreakdown.java | 189 ++++++++++++------ .../query/ConcurrentQueryProfileTree.java | 11 +- .../query/ConcurrentQueryProfiler.java | 4 +- .../query/InternalQueryProfileTree.java | 5 +- .../search/profile/query/ProfileScorer.java | 2 +- .../search/profile/query/ProfileWeight.java | 4 +- .../profile/query/QueryProfileBreakdown.java | 14 +- .../search/profile/query/QueryProfiler.java | 2 +- .../AbstractProfileBreakdownTests.java | 61 ++++++ .../search/profile/ProfileMetricTests.java | 49 +++++ .../opensearch/search/profile/TimerTests.java | 17 +- .../ConcurrentQueryProfileBreakdownTests.java | 128 ++++++++---- .../query/ConcurrentQueryProfilerTests.java | 6 +- .../profile/query/ProfileScorerTests.java | 5 +- 28 files changed, 530 insertions(+), 176 deletions(-) create mode 100644 server/src/main/java/org/opensearch/search/profile/ProfileMetric.java create mode 100644 server/src/main/java/org/opensearch/search/profile/ProfileMetricUtil.java create mode 100644 server/src/test/java/org/opensearch/search/profile/AbstractProfileBreakdownTests.java create mode 100644 server/src/test/java/org/opensearch/search/profile/ProfileMetricTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index a4337a26966cd..bcfb8e3195134 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Introduce a new pull-based ingestion plugin for file-based indexing (for local testing) ([#18591](https://github.com/opensearch-project/OpenSearch/pull/18591)) - Add support for search pipeline in search and msearch template ([#18564](https://github.com/opensearch-project/OpenSearch/pull/18564)) - Add BooleanQuery rewrite moving constant-scoring must clauses to filter clauses ([#18510](https://github.com/opensearch-project/OpenSearch/issues/18510)) +- Add support for non-timing info in profiler ([#18460](https://github.com/opensearch-project/OpenSearch/issues/18460)) ### Changed - Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570)) diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java index bf570a0dd0d60..4a5159681d07f 100644 --- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java @@ -213,7 +213,7 @@ public Weight createWeight(Query query, ScoreMode scoreMode, float boost) throws // createWeight() is called for each query in the tree, so we tell the queryProfiler // each invocation so that it can build an internal representation of the query // tree - ContextualProfileBreakdown profile = profiler.getQueryBreakdown(query); + ContextualProfileBreakdown profile = profiler.getQueryBreakdown(query); Timer timer = profile.getTimer(QueryTimingType.CREATE_WEIGHT); timer.start(); final Weight weight; diff --git a/server/src/main/java/org/opensearch/search/profile/AbstractInternalProfileTree.java b/server/src/main/java/org/opensearch/search/profile/AbstractInternalProfileTree.java index 904b04b249b1b..f067064cc56f5 100644 --- a/server/src/main/java/org/opensearch/search/profile/AbstractInternalProfileTree.java +++ b/server/src/main/java/org/opensearch/search/profile/AbstractInternalProfileTree.java @@ -45,7 +45,7 @@ * * @opensearch.internal */ -public abstract class AbstractInternalProfileTree, E> { +public abstract class AbstractInternalProfileTree { protected ArrayList breakdowns; /** Maps the Query to it's list of children. This is basically the dependency tree */ diff --git a/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java index 4a1563e7cdce9..1bacc26eaa024 100644 --- a/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java +++ b/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java @@ -32,9 +32,12 @@ package org.opensearch.search.profile; +import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.Map; +import java.util.TreeMap; +import java.util.function.Supplier; +import java.util.stream.Collectors; import static java.util.Collections.emptyMap; @@ -45,46 +48,46 @@ * * @opensearch.internal */ -public abstract class AbstractProfileBreakdown> { +public abstract class AbstractProfileBreakdown { - /** - * The accumulated timings for this query node - */ - protected final Timer[] timings; - protected final T[] timingTypes; - public static final String TIMING_TYPE_COUNT_SUFFIX = "_count"; - public static final String TIMING_TYPE_START_TIME_SUFFIX = "_start_time"; + protected final Map metrics; /** Sole constructor. */ - public AbstractProfileBreakdown(Class clazz) { - this.timingTypes = clazz.getEnumConstants(); - timings = new Timer[timingTypes.length]; - for (int i = 0; i < timings.length; ++i) { - timings[i] = new Timer(); - } + public AbstractProfileBreakdown(Collection> metricSuppliers) { + this.metrics = metricSuppliers.stream().map(Supplier::get).collect(Collectors.toMap(ProfileMetric::getName, metric -> metric)); } - public Timer getTimer(T timing) { - return timings[timing.ordinal()]; + public Timer getTimer(Enum type) { + ProfileMetric metric = metrics.get(type.toString()); + assert metric instanceof Timer : "Metric " + type + " is not a timer"; + return (Timer) metric; } - public void setTimer(T timing, Timer timer) { - timings[timing.ordinal()] = timer; + public ProfileMetric getMetric(String name) { + return metrics.get(name); } /** - * Build a timing count breakdown for current instance + * Build a breakdown for current instance */ public Map toBreakdownMap() { - Map map = new HashMap<>(this.timings.length * 3); - for (T timingType : this.timingTypes) { - map.put(timingType.toString(), this.timings[timingType.ordinal()].getApproximateTiming()); - map.put(timingType + TIMING_TYPE_COUNT_SUFFIX, this.timings[timingType.ordinal()].getCount()); - map.put(timingType + TIMING_TYPE_START_TIME_SUFFIX, this.timings[timingType.ordinal()].getEarliestTimerStartTime()); + Map map = new TreeMap<>(); + for (Map.Entry entry : metrics.entrySet()) { + map.putAll(entry.getValue().toBreakdownMap()); } return Collections.unmodifiableMap(map); } + public long toNodeTime() { + long total = 0; + for (Map.Entry entry : metrics.entrySet()) { + if (entry.getValue() instanceof Timer t) { + total += t.getApproximateTiming(); + } + } + return total; + } + /** * Fetch extra debugging information. */ @@ -92,11 +95,4 @@ public Map toDebugMap() { return emptyMap(); } - public long toNodeTime() { - long total = 0; - for (T timingType : timingTypes) { - total += timings[timingType.ordinal()].getApproximateTiming(); - } - return total; - } } diff --git a/server/src/main/java/org/opensearch/search/profile/AbstractProfiler.java b/server/src/main/java/org/opensearch/search/profile/AbstractProfiler.java index 4db1cb87a231d..0527bb44e16d4 100644 --- a/server/src/main/java/org/opensearch/search/profile/AbstractProfiler.java +++ b/server/src/main/java/org/opensearch/search/profile/AbstractProfiler.java @@ -39,7 +39,7 @@ * * @opensearch.internal */ -public class AbstractProfiler, E> { +public abstract class AbstractProfiler { protected final AbstractInternalProfileTree profileTree; diff --git a/server/src/main/java/org/opensearch/search/profile/ContextualProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/ContextualProfileBreakdown.java index 3fe621321c8ad..2ca88f5cabc85 100644 --- a/server/src/main/java/org/opensearch/search/profile/ContextualProfileBreakdown.java +++ b/server/src/main/java/org/opensearch/search/profile/ContextualProfileBreakdown.java @@ -11,8 +11,10 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Collector; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * Provide contextual profile breakdowns which are associated with freestyle context. Used when concurrent @@ -20,9 +22,10 @@ * * @opensearch.internal */ -public abstract class ContextualProfileBreakdown> extends AbstractProfileBreakdown { - public ContextualProfileBreakdown(Class clazz) { - super(clazz); +public abstract class ContextualProfileBreakdown extends AbstractProfileBreakdown { + + public ContextualProfileBreakdown(Collection> metrics) { + super(metrics); } /** @@ -30,7 +33,7 @@ public ContextualProfileBreakdown(Class clazz) { * @param context freestyle context * @return contextual profile breakdown instance */ - public abstract AbstractProfileBreakdown context(Object context); + public abstract AbstractProfileBreakdown context(Object context); public void associateCollectorToLeaves(Collector collector, LeafReaderContext leaf) {} diff --git a/server/src/main/java/org/opensearch/search/profile/ProfileMetric.java b/server/src/main/java/org/opensearch/search/profile/ProfileMetric.java new file mode 100644 index 0000000000000..36bb9b300f3b3 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/profile/ProfileMetric.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.profile; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.Map; + +/** + * A metric for profiling. + */ +@ExperimentalApi +public abstract class ProfileMetric { + + private final String name; + + public ProfileMetric(String name) { + this.name = name; + } + + /** + * + * @return name of the metric + */ + public String getName() { + return name; + } + + /** + * + * @return map representation of breakdown + */ + abstract public Map toBreakdownMap(); +} diff --git a/server/src/main/java/org/opensearch/search/profile/ProfileMetricUtil.java b/server/src/main/java/org/opensearch/search/profile/ProfileMetricUtil.java new file mode 100644 index 0000000000000..248e29fc0383f --- /dev/null +++ b/server/src/main/java/org/opensearch/search/profile/ProfileMetricUtil.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.profile; + +import org.opensearch.search.profile.aggregation.AggregationTimingType; +import org.opensearch.search.profile.query.QueryTimingType; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.function.Supplier; + +/** + * Utility class to provide profile metrics to breakdowns. + */ +public class ProfileMetricUtil { + + public static Collection> getDefaultQueryProfileMetrics() { + Collection> metrics = new ArrayList<>(); + for (QueryTimingType type : QueryTimingType.values()) { + metrics.add(() -> new Timer(type.toString())); + } + return metrics; + } + + public static Collection> getAggregationProfileMetrics() { + Collection> metrics = new ArrayList<>(); + for (AggregationTimingType type : AggregationTimingType.values()) { + metrics.add(() -> new Timer(type.toString())); + } + return metrics; + } +} diff --git a/server/src/main/java/org/opensearch/search/profile/ProfileResult.java b/server/src/main/java/org/opensearch/search/profile/ProfileResult.java index 2c0d2cf3ba78a..7969a9e5890bf 100644 --- a/server/src/main/java/org/opensearch/search/profile/ProfileResult.java +++ b/server/src/main/java/org/opensearch/search/profile/ProfileResult.java @@ -271,7 +271,7 @@ static void removeStartTimeFields(Map modifiedBreakdown) { Iterator> iterator = modifiedBreakdown.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry entry = iterator.next(); - if (entry.getKey().endsWith(AbstractProfileBreakdown.TIMING_TYPE_START_TIME_SUFFIX)) { + if (entry.getKey().endsWith(Timer.TIMING_TYPE_START_TIME_SUFFIX)) { iterator.remove(); } } diff --git a/server/src/main/java/org/opensearch/search/profile/Profilers.java b/server/src/main/java/org/opensearch/search/profile/Profilers.java index 75337f89e67ca..5aaf1d670313e 100644 --- a/server/src/main/java/org/opensearch/search/profile/Profilers.java +++ b/server/src/main/java/org/opensearch/search/profile/Profilers.java @@ -79,7 +79,7 @@ public QueryProfiler addQueryProfiler() { /** Get the current profiler. */ public QueryProfiler getCurrentQueryProfiler() { - return queryProfilers.get(queryProfilers.size() - 1); + return queryProfilers.getLast(); } /** Return the list of all created {@link QueryProfiler}s so far. */ diff --git a/server/src/main/java/org/opensearch/search/profile/Timer.java b/server/src/main/java/org/opensearch/search/profile/Timer.java index 864c689cf7fa0..673d05bfdca91 100644 --- a/server/src/main/java/org/opensearch/search/profile/Timer.java +++ b/server/src/main/java/org/opensearch/search/profile/Timer.java @@ -32,6 +32,9 @@ package org.opensearch.search.profile; +import java.util.HashMap; +import java.util.Map; + /** Helps measure how much time is spent running some methods. * The {@link #start()} and {@link #stop()} methods should typically be called * in a try/finally clause with {@link #start()} being called right before the @@ -48,16 +51,19 @@ * * @opensearch.internal */ -public class Timer { +public class Timer extends ProfileMetric { + public static final String TIMING_TYPE_COUNT_SUFFIX = "_count"; + public static final String TIMING_TYPE_START_TIME_SUFFIX = "_start_time"; private boolean doTiming; private long timing, count, lastCount, start, earliestTimerStartTime; - public Timer() { - this(0, 0, 0, 0, 0); + public Timer(String name) { + super(name); } - public Timer(long timing, long count, long lastCount, long start, long earliestTimerStartTime) { + public Timer(long timing, long count, long lastCount, long start, long earliestTimerStartTime, String name) { + super(name); this.timing = timing; this.count = count; this.lastCount = lastCount; @@ -131,4 +137,13 @@ public final long getApproximateTiming() { } return timing; } + + @Override + public Map toBreakdownMap() { + Map map = new HashMap<>(); + map.put(getName(), getApproximateTiming()); + map.put(getName() + TIMING_TYPE_COUNT_SUFFIX, getCount()); + map.put(getName() + TIMING_TYPE_START_TIME_SUFFIX, getEarliestTimerStartTime()); + return map; + } } diff --git a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileBreakdown.java index 8642f0da4a90b..3c99b041a423c 100644 --- a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileBreakdown.java +++ b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileBreakdown.java @@ -34,9 +34,13 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.search.profile.AbstractProfileBreakdown; +import org.opensearch.search.profile.ProfileMetric; +import org.opensearch.search.profile.ProfileMetricUtil; +import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.function.Supplier; import static java.util.Collections.unmodifiableMap; @@ -46,11 +50,15 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class AggregationProfileBreakdown extends AbstractProfileBreakdown { +public class AggregationProfileBreakdown extends AbstractProfileBreakdown { private final Map extra = new HashMap<>(); public AggregationProfileBreakdown() { - super(AggregationTimingType.class); + this(ProfileMetricUtil.getAggregationProfileMetrics()); + } + + public AggregationProfileBreakdown(Collection> timers) { + super(timers); } /** @@ -64,4 +72,5 @@ public void addDebugInfo(String key, Object value) { public Map toDebugMap() { return unmodifiableMap(extra); } + } diff --git a/server/src/main/java/org/opensearch/search/profile/aggregation/ConcurrentAggregationProfiler.java b/server/src/main/java/org/opensearch/search/profile/aggregation/ConcurrentAggregationProfiler.java index deed68c535cf9..593359cd5c22c 100644 --- a/server/src/main/java/org/opensearch/search/profile/aggregation/ConcurrentAggregationProfiler.java +++ b/server/src/main/java/org/opensearch/search/profile/aggregation/ConcurrentAggregationProfiler.java @@ -13,8 +13,8 @@ package org.opensearch.search.profile.aggregation; -import org.opensearch.search.profile.AbstractProfileBreakdown; import org.opensearch.search.profile.ProfileResult; +import org.opensearch.search.profile.Timer; import java.util.HashMap; import java.util.LinkedList; @@ -31,7 +31,7 @@ public class ConcurrentAggregationProfiler extends AggregationProfiler { private static final String MAX_PREFIX = "max_"; private static final String MIN_PREFIX = "min_"; private static final String AVG_PREFIX = "avg_"; - private static final String START_TIME_KEY = AggregationTimingType.INITIALIZE + AbstractProfileBreakdown.TIMING_TYPE_START_TIME_SUFFIX; + private static final String START_TIME_KEY = AggregationTimingType.INITIALIZE + Timer.TIMING_TYPE_START_TIME_SUFFIX; private static final String[] breakdownCountStatsTypes = { "build_leaf_collector_count", "collect_count" }; @Override @@ -82,8 +82,7 @@ private List reduceProfileResultsTree(List profile // Profiled breakdown total time for (AggregationTimingType timingType : AggregationTimingType.values()) { String breakdownTimingType = timingType.toString(); - Long startTime = profileResult.getTimeBreakdown() - .get(breakdownTimingType + AbstractProfileBreakdown.TIMING_TYPE_START_TIME_SUFFIX); + Long startTime = profileResult.getTimeBreakdown().get(breakdownTimingType + Timer.TIMING_TYPE_START_TIME_SUFFIX); Long endTime = startTime + profileResult.getTimeBreakdown().get(breakdownTimingType); minSliceStartTimeMap.put( breakdownTimingType, @@ -103,7 +102,7 @@ private List reduceProfileResultsTree(List profile // Profiled breakdown count for (AggregationTimingType timingType : AggregationTimingType.values()) { String breakdownType = timingType.toString(); - String breakdownTypeCount = breakdownType + AbstractProfileBreakdown.TIMING_TYPE_COUNT_SUFFIX; + String breakdownTypeCount = breakdownType + Timer.TIMING_TYPE_COUNT_SUFFIX; breakdown.put( breakdownTypeCount, breakdown.getOrDefault(breakdownTypeCount, 0L) + profileResult.getTimeBreakdown().get(breakdownTypeCount) diff --git a/server/src/main/java/org/opensearch/search/profile/query/AbstractQueryProfileTree.java b/server/src/main/java/org/opensearch/search/profile/query/AbstractQueryProfileTree.java index 2f5d632ee2d87..2541bd3887f13 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/AbstractQueryProfileTree.java +++ b/server/src/main/java/org/opensearch/search/profile/query/AbstractQueryProfileTree.java @@ -20,7 +20,7 @@ * * @opensearch.internal */ -public abstract class AbstractQueryProfileTree extends AbstractInternalProfileTree, Query> { +public abstract class AbstractQueryProfileTree extends AbstractInternalProfileTree { /** Rewrite time */ private long rewriteTime; @@ -64,4 +64,6 @@ public void stopAndAddRewriteTime() { public long getRewriteTime() { return rewriteTime; } + + protected abstract ContextualProfileBreakdown createProfileBreakdown(); } diff --git a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java index c017b15bc6d3c..8ee0f0b8ac7e1 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java +++ b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java @@ -13,13 +13,23 @@ import org.opensearch.OpenSearchException; import org.opensearch.search.profile.AbstractProfileBreakdown; import org.opensearch.search.profile.ContextualProfileBreakdown; +import org.opensearch.search.profile.ProfileMetric; +import org.opensearch.search.profile.Timer; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +import static org.opensearch.search.profile.Timer.TIMING_TYPE_COUNT_SUFFIX; +import static org.opensearch.search.profile.Timer.TIMING_TYPE_START_TIME_SUFFIX; /** * A record of timings for the various operations that may happen during query execution. @@ -28,7 +38,7 @@ * * @opensearch.internal */ -public final class ConcurrentQueryProfileBreakdown extends ContextualProfileBreakdown { +public final class ConcurrentQueryProfileBreakdown extends ContextualProfileBreakdown { static final String SLICE_END_TIME_SUFFIX = "_slice_end_time"; static final String SLICE_START_TIME_SUFFIX = "_slice_start_time"; static final String MAX_PREFIX = "max_"; @@ -40,26 +50,32 @@ public final class ConcurrentQueryProfileBreakdown extends ContextualProfileBrea private long avgSliceNodeTime = 0L; // keep track of all breakdown timings per segment. package-private for testing - private final Map> contexts = new ConcurrentHashMap<>(); + private final Map contexts = new ConcurrentHashMap<>(); // represents slice to leaves mapping as for each slice a unique collector instance is created private final Map> sliceCollectorsToLeaves = new ConcurrentHashMap<>(); - /** Sole constructor. */ - public ConcurrentQueryProfileBreakdown() { - super(QueryTimingType.class); + private final Collection> metricSuppliers; + private final Set timingMetrics; + private final Set nonTimingMetrics; + + public ConcurrentQueryProfileBreakdown(Collection> metricSuppliers) { + super(metricSuppliers); + this.metricSuppliers = metricSuppliers; + this.timingMetrics = getTimingMetrics(); + this.nonTimingMetrics = getNonTimingMetrics(); } @Override - public AbstractProfileBreakdown context(Object context) { + public AbstractProfileBreakdown context(Object context) { // See please https://bugs.openjdk.java.net/browse/JDK-8161372 - final AbstractProfileBreakdown profile = contexts.get(context); + final AbstractProfileBreakdown profile = contexts.get(context); if (profile != null) { return profile; } - return contexts.computeIfAbsent(context, ctx -> new QueryProfileBreakdown()); + return contexts.computeIfAbsent(context, ctx -> new QueryProfileBreakdown(metricSuppliers)); } @Override @@ -87,12 +103,12 @@ public Map toBreakdownMap() { assert contexts.size() == 1 : "Unexpected size: " + contexts.size() + " of leaves breakdown in ConcurrentQueryProfileBreakdown of rewritten query for a leaf."; - AbstractProfileBreakdown breakdown = contexts.values().iterator().next(); + AbstractProfileBreakdown breakdown = contexts.values().iterator().next(); queryNodeTime = breakdown.toNodeTime() + createWeightTime; maxSliceNodeTime = 0L; minSliceNodeTime = 0L; avgSliceNodeTime = 0L; - Map queryBreakdownMap = new HashMap<>(breakdown.toBreakdownMap()); + Map queryBreakdownMap = new TreeMap<>(breakdown.toBreakdownMap()); queryBreakdownMap.put(QueryTimingType.CREATE_WEIGHT.toString(), createWeightTime); queryBreakdownMap.put(QueryTimingType.CREATE_WEIGHT + TIMING_TYPE_COUNT_SUFFIX, 1L); return queryBreakdownMap; @@ -110,19 +126,19 @@ public Map toBreakdownMap() { * default breakdown map. */ private Map buildDefaultQueryBreakdownMap(long createWeightTime) { - final Map concurrentQueryBreakdownMap = new HashMap<>(); + final Map concurrentQueryBreakdownMap = new TreeMap<>(); for (QueryTimingType timingType : QueryTimingType.values()) { final String timingTypeKey = timingType.toString(); - final String timingTypeCountKey = timingTypeKey + TIMING_TYPE_COUNT_SUFFIX; + final String timingTypeCountKey = timingType + TIMING_TYPE_COUNT_SUFFIX; if (timingType.equals(QueryTimingType.CREATE_WEIGHT)) { concurrentQueryBreakdownMap.put(timingTypeKey, createWeightTime); concurrentQueryBreakdownMap.put(timingTypeCountKey, 1L); continue; } - final String maxBreakdownTypeTime = MAX_PREFIX + timingTypeKey; - final String minBreakdownTypeTime = MIN_PREFIX + timingTypeKey; - final String avgBreakdownTypeTime = AVG_PREFIX + timingTypeKey; + final String maxBreakdownTypeTime = MAX_PREFIX + timingType; + final String minBreakdownTypeTime = MIN_PREFIX + timingType; + final String avgBreakdownTypeTime = AVG_PREFIX + timingType; final String maxBreakdownTypeCount = MAX_PREFIX + timingTypeCountKey; final String minBreakdownTypeCount = MIN_PREFIX + timingTypeCountKey; final String avgBreakdownTypeCount = AVG_PREFIX + timingTypeCountKey; @@ -156,8 +172,9 @@ Map> buildSliceLevelBreakdown() { // max slice end time across all timing types long sliceMaxEndTime = Long.MIN_VALUE; long sliceMinStartTime = Long.MAX_VALUE; - for (QueryTimingType timingType : QueryTimingType.values()) { - if (timingType.equals(QueryTimingType.CREATE_WEIGHT)) { + + for (String timingType : timingMetrics) { + if (timingType.equals(QueryTimingType.CREATE_WEIGHT.toString())) { // do nothing for create weight as that is query level time and not slice level continue; } @@ -210,9 +227,7 @@ Map> buildSliceLevelBreakdown() { ); // compute the sliceEndTime for timingType using max of endTime across slice leaves - final long sliceLeafTimingTypeEndTime = sliceLeafTimingTypeStartTime + currentSliceLeafBreakdownMap.get( - timingType.toString() - ); + final long sliceLeafTimingTypeEndTime = sliceLeafTimingTypeStartTime + currentSliceLeafBreakdownMap.get(timingType); currentSliceBreakdown.compute( timingTypeSliceEndTimeKey, (key, value) -> (value == null) ? sliceLeafTimingTypeEndTime : Math.max(value, sliceLeafTimingTypeEndTime) @@ -235,13 +250,28 @@ Map> buildSliceLevelBreakdown() { sliceMinStartTime = Math.min(sliceMinStartTime, currentSliceStartTime); // compute total time for each timing type at slice level using sliceEndTime and sliceStartTime currentSliceBreakdown.put( - timingType.toString(), + timingType, currentSliceBreakdown.getOrDefault(timingTypeSliceEndTimeKey, 0L) - currentSliceBreakdown.getOrDefault( timingTypeSliceStartTimeKey, 0L ) ); } + + for (String metric : nonTimingMetrics) { + for (LeafReaderContext sliceLeaf : slice.getValue()) { + if (!contexts.containsKey(sliceLeaf)) { + continue; + } + final Map currentSliceLeafBreakdownMap = contexts.get(sliceLeaf).toBreakdownMap(); + final long sliceLeafMetricValue = currentSliceLeafBreakdownMap.get(metric); + currentSliceBreakdown.compute( + metric, + (key, value) -> (value == null) ? sliceLeafMetricValue : value + sliceLeafMetricValue + ); + } + } + // currentSliceNodeTime does not include the create weight time, as that is computed in non-concurrent part long currentSliceNodeTime; if (sliceMinStartTime == Long.MAX_VALUE && sliceMaxEndTime == Long.MIN_VALUE) { @@ -284,17 +314,27 @@ public Map buildQueryBreakdownMap( long createWeightTime, long createWeightStartTime ) { - final Map queryBreakdownMap = new HashMap<>(); + final Map queryBreakdownMap = new TreeMap<>(); long queryEndTime = Long.MIN_VALUE; - for (QueryTimingType queryTimingType : QueryTimingType.values()) { - final String timingTypeKey = queryTimingType.toString(); - final String timingTypeCountKey = timingTypeKey + TIMING_TYPE_COUNT_SUFFIX; - final String sliceEndTimeForTimingType = timingTypeKey + SLICE_END_TIME_SUFFIX; - final String sliceStartTimeForTimingType = timingTypeKey + SLICE_START_TIME_SUFFIX; - - final String maxBreakdownTypeTime = MAX_PREFIX + timingTypeKey; - final String minBreakdownTypeTime = MIN_PREFIX + timingTypeKey; - final String avgBreakdownTypeTime = AVG_PREFIX + timingTypeKey; + + // the create weight time is computed at the query level and is called only once per query + queryBreakdownMap.put(QueryTimingType.CREATE_WEIGHT + TIMING_TYPE_COUNT_SUFFIX, 1L); + queryBreakdownMap.put(QueryTimingType.CREATE_WEIGHT.toString(), createWeightTime); + + for (String metric : timingMetrics) { + + if (metric.equals(QueryTimingType.CREATE_WEIGHT.toString())) { + // create weight time is computed at query level and is called only once per query + continue; + } + + final String timingTypeCountKey = metric + TIMING_TYPE_COUNT_SUFFIX; + final String sliceEndTimeForTimingType = metric + SLICE_END_TIME_SUFFIX; + final String sliceStartTimeForTimingType = metric + SLICE_START_TIME_SUFFIX; + + final String maxBreakdownTypeTime = MAX_PREFIX + metric; + final String minBreakdownTypeTime = MIN_PREFIX + metric; + final String avgBreakdownTypeTime = AVG_PREFIX + metric; final String maxBreakdownTypeCount = MAX_PREFIX + timingTypeCountKey; final String minBreakdownTypeCount = MIN_PREFIX + timingTypeCountKey; final String avgBreakdownTypeCount = AVG_PREFIX + timingTypeCountKey; @@ -303,43 +343,19 @@ public Map buildQueryBreakdownMap( long queryTimingTypeStartTime = Long.MAX_VALUE; long queryTimingTypeCount = 0L; - // the create weight time is computed at the query level and is called only once per query - if (queryTimingType == QueryTimingType.CREATE_WEIGHT) { - queryBreakdownMap.put(timingTypeCountKey, 1L); - queryBreakdownMap.put(timingTypeKey, createWeightTime); - continue; - } - // for all other timing types, we will compute min/max/avg/total across slices for (Map.Entry> sliceBreakdown : sliceLevelBreakdowns.entrySet()) { - long sliceBreakdownTypeTime = sliceBreakdown.getValue().getOrDefault(timingTypeKey, 0L); + long sliceBreakdownTypeTime = sliceBreakdown.getValue().getOrDefault(metric, 0L); long sliceBreakdownTypeCount = sliceBreakdown.getValue().getOrDefault(timingTypeCountKey, 0L); // compute max/min/avg TimingType time across slices - queryBreakdownMap.compute( - maxBreakdownTypeTime, - (key, value) -> (value == null) ? sliceBreakdownTypeTime : Math.max(sliceBreakdownTypeTime, value) - ); - queryBreakdownMap.compute( - minBreakdownTypeTime, - (key, value) -> (value == null) ? sliceBreakdownTypeTime : Math.min(sliceBreakdownTypeTime, value) - ); - queryBreakdownMap.compute( - avgBreakdownTypeTime, - (key, value) -> (value == null) ? sliceBreakdownTypeTime : sliceBreakdownTypeTime + value - ); - + addStatsToMap(queryBreakdownMap, maxBreakdownTypeTime, minBreakdownTypeTime, avgBreakdownTypeTime, sliceBreakdownTypeTime); // compute max/min/avg TimingType count across slices - queryBreakdownMap.compute( + addStatsToMap( + queryBreakdownMap, maxBreakdownTypeCount, - (key, value) -> (value == null) ? sliceBreakdownTypeCount : Math.max(sliceBreakdownTypeCount, value) - ); - queryBreakdownMap.compute( minBreakdownTypeCount, - (key, value) -> (value == null) ? sliceBreakdownTypeCount : Math.min(sliceBreakdownTypeCount, value) - ); - queryBreakdownMap.compute( avgBreakdownTypeCount, - (key, value) -> (value == null) ? sliceBreakdownTypeCount : sliceBreakdownTypeCount + value + sliceBreakdownTypeCount ); // only modify the start/end time of the TimingType if the slice used the timer @@ -360,7 +376,7 @@ public Map buildQueryBreakdownMap( if (queryTimingTypeCount > 0L && (queryTimingTypeStartTime == Long.MAX_VALUE || queryTimingTypeEndTime == Long.MIN_VALUE)) { throw new OpenSearchException( "Unexpected timing type [" - + timingTypeKey + + metric + "] start [" + queryTimingTypeStartTime + "] or end time [" @@ -368,13 +384,33 @@ public Map buildQueryBreakdownMap( + "] computed across slices for profile results" ); } - queryBreakdownMap.put(timingTypeKey, (queryTimingTypeCount > 0L) ? queryTimingTypeEndTime - queryTimingTypeStartTime : 0L); + queryBreakdownMap.put(metric, (queryTimingTypeCount > 0L) ? queryTimingTypeEndTime - queryTimingTypeStartTime : 0L); queryBreakdownMap.put(timingTypeCountKey, queryTimingTypeCount); queryBreakdownMap.compute(avgBreakdownTypeTime, (key, value) -> (value == null) ? 0L : value / sliceLevelBreakdowns.size()); queryBreakdownMap.compute(avgBreakdownTypeCount, (key, value) -> (value == null) ? 0L : value / sliceLevelBreakdowns.size()); // compute query end time using max of query end time across all timing types queryEndTime = Math.max(queryEndTime, queryTimingTypeEndTime); } + + for (String metric : nonTimingMetrics) { + + final String maxBreakdownTypeTime = MAX_PREFIX + metric; + final String minBreakdownTypeTime = MIN_PREFIX + metric; + final String avgBreakdownTypeTime = AVG_PREFIX + metric; + + long totalBreakdownValue = 0L; + + // for all other timing types, we will compute min/max/avg/total across slices + for (Map.Entry> sliceBreakdown : sliceLevelBreakdowns.entrySet()) { + long sliceBreakdownValue = sliceBreakdown.getValue().getOrDefault(metric, 0L); + // compute max/min/avg TimingType time across slices + addStatsToMap(queryBreakdownMap, maxBreakdownTypeTime, minBreakdownTypeTime, avgBreakdownTypeTime, sliceBreakdownValue); + totalBreakdownValue += sliceBreakdownValue; + } + queryBreakdownMap.put(metric, totalBreakdownValue); + queryBreakdownMap.compute(avgBreakdownTypeTime, (key, value) -> (value == null) ? 0L : value / sliceLevelBreakdowns.size()); + } + if (queryEndTime == Long.MIN_VALUE) { throw new OpenSearchException("Unexpected error while computing the query end time across slices in profile result"); } @@ -382,6 +418,32 @@ public Map buildQueryBreakdownMap( return queryBreakdownMap; } + private void addStatsToMap(Map queryBreakdownMap, String maxKey, String minKey, String avgKey, long sliceValue) { + queryBreakdownMap.compute(maxKey, (key, value) -> (value == null) ? sliceValue : Math.max(sliceValue, value)); + queryBreakdownMap.compute(minKey, (key, value) -> (value == null) ? sliceValue : Math.min(sliceValue, value)); + queryBreakdownMap.compute(avgKey, (key, value) -> (value == null) ? sliceValue : (value + sliceValue)); + } + + private Set getTimingMetrics() { + Set result = new HashSet<>(); + for (Map.Entry entry : metrics.entrySet()) { + if (entry.getValue() instanceof org.opensearch.search.profile.Timer) { + result.add(entry.getKey()); + } + } + return result; + } + + private Set getNonTimingMetrics() { + Set result = new HashSet<>(); + for (Map.Entry entry : metrics.entrySet()) { + if (!(entry.getValue() instanceof Timer)) { + result.add(entry.getKey()); + } + } + return result; + } + @Override public long toNodeTime() { return queryNodeTime; @@ -403,7 +465,7 @@ Map> getSliceCollectorsToLeaves() { } // used by tests - Map> getContexts() { + Map getContexts() { return contexts; } @@ -418,4 +480,5 @@ long getMinSliceNodeTime() { long getAvgSliceNodeTime() { return avgSliceNodeTime; } + } diff --git a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileTree.java b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileTree.java index 4e54178c3b4fb..ae549a5e9cbe2 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileTree.java +++ b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileTree.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Collector; import org.opensearch.search.profile.ContextualProfileBreakdown; +import org.opensearch.search.profile.ProfileMetricUtil; import org.opensearch.search.profile.ProfileResult; import java.util.List; @@ -24,15 +25,15 @@ public class ConcurrentQueryProfileTree extends AbstractQueryProfileTree { @Override - protected ContextualProfileBreakdown createProfileBreakdown() { - return new ConcurrentQueryProfileBreakdown(); + protected ContextualProfileBreakdown createProfileBreakdown() { + return new ConcurrentQueryProfileBreakdown(ProfileMetricUtil.getDefaultQueryProfileMetrics()); } @Override protected ProfileResult createProfileResult( String type, String description, - ContextualProfileBreakdown breakdown, + ContextualProfileBreakdown breakdown, List childrenProfileResults ) { assert breakdown instanceof ConcurrentQueryProfileBreakdown; @@ -62,7 +63,7 @@ protected ProfileResult createProfileResult( @Override public List getTree() { for (Integer root : roots) { - final ContextualProfileBreakdown parentBreakdown = breakdowns.get(root); + final ContextualProfileBreakdown parentBreakdown = breakdowns.get(root); assert parentBreakdown instanceof ConcurrentQueryProfileBreakdown; final Map> parentCollectorToLeaves = ((ConcurrentQueryProfileBreakdown) parentBreakdown) .getSliceCollectorsToLeaves(); @@ -82,7 +83,7 @@ private void updateCollectorToLeavesForChildBreakdowns(Integer parentToken, Map< final List children = tree.get(parentToken); if (children != null) { for (Integer currentChild : children) { - final ContextualProfileBreakdown currentChildBreakdown = breakdowns.get(currentChild); + final ContextualProfileBreakdown currentChildBreakdown = breakdowns.get(currentChild); currentChildBreakdown.associateCollectorsToLeaves(collectorToLeaves); updateCollectorToLeavesForChildBreakdowns(currentChild, collectorToLeaves); } diff --git a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfiler.java b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfiler.java index e02e562d766a0..3bce0ecd5c252 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfiler.java +++ b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfiler.java @@ -45,7 +45,7 @@ public ConcurrentQueryProfiler(AbstractQueryProfileTree profileTree) { } @Override - public ContextualProfileBreakdown getQueryBreakdown(Query query) { + public ContextualProfileBreakdown getQueryBreakdown(Query query) { ConcurrentQueryProfileTree profileTree = threadToProfileTree.computeIfAbsent( getCurrentThreadId(), k -> new ConcurrentQueryProfileTree() @@ -81,7 +81,7 @@ public List getTree() { */ @Override public void startRewriteTime() { - Timer rewriteTimer = new Timer(); + Timer rewriteTimer = new Timer("rewrite_timer"); threadToRewriteTimers.computeIfAbsent(getCurrentThreadId(), k -> new LinkedList<>()).add(rewriteTimer); rewriteTimer.start(); } diff --git a/server/src/main/java/org/opensearch/search/profile/query/InternalQueryProfileTree.java b/server/src/main/java/org/opensearch/search/profile/query/InternalQueryProfileTree.java index 1ed367f094fb7..14dd0bc853fa0 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/InternalQueryProfileTree.java +++ b/server/src/main/java/org/opensearch/search/profile/query/InternalQueryProfileTree.java @@ -33,6 +33,7 @@ package org.opensearch.search.profile.query; import org.opensearch.search.profile.ContextualProfileBreakdown; +import org.opensearch.search.profile.ProfileMetricUtil; import org.opensearch.search.profile.ProfileResult; /** @@ -43,7 +44,7 @@ public class InternalQueryProfileTree extends AbstractQueryProfileTree { @Override - protected ContextualProfileBreakdown createProfileBreakdown() { - return new QueryProfileBreakdown(); + protected ContextualProfileBreakdown createProfileBreakdown() { + return new QueryProfileBreakdown(ProfileMetricUtil.getDefaultQueryProfileMetrics()); } } diff --git a/server/src/main/java/org/opensearch/search/profile/query/ProfileScorer.java b/server/src/main/java/org/opensearch/search/profile/query/ProfileScorer.java index 28b693ee03ad5..b78ecf4501ae1 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/ProfileScorer.java +++ b/server/src/main/java/org/opensearch/search/profile/query/ProfileScorer.java @@ -54,7 +54,7 @@ final class ProfileScorer extends Scorer { private final Timer scoreTimer, nextDocTimer, advanceTimer, matchTimer, shallowAdvanceTimer, computeMaxScoreTimer, setMinCompetitiveScoreTimer; - ProfileScorer(Scorer scorer, AbstractProfileBreakdown profile) throws IOException { + ProfileScorer(Scorer scorer, AbstractProfileBreakdown profile) throws IOException { this.scorer = scorer; scoreTimer = profile.getTimer(QueryTimingType.SCORE); nextDocTimer = profile.getTimer(QueryTimingType.NEXT_DOC); diff --git a/server/src/main/java/org/opensearch/search/profile/query/ProfileWeight.java b/server/src/main/java/org/opensearch/search/profile/query/ProfileWeight.java index f190a9734c1a5..c200ebc6ba8f0 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/ProfileWeight.java +++ b/server/src/main/java/org/opensearch/search/profile/query/ProfileWeight.java @@ -54,9 +54,9 @@ public final class ProfileWeight extends Weight { private final Weight subQueryWeight; - private final ContextualProfileBreakdown profile; + private final ContextualProfileBreakdown profile; - public ProfileWeight(Query query, Weight subQueryWeight, ContextualProfileBreakdown profile) throws IOException { + public ProfileWeight(Query query, Weight subQueryWeight, ContextualProfileBreakdown profile) throws IOException { super(query); this.subQueryWeight = subQueryWeight; this.profile = profile; diff --git a/server/src/main/java/org/opensearch/search/profile/query/QueryProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/query/QueryProfileBreakdown.java index 3514a80e39d85..873c7dc22df65 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/QueryProfileBreakdown.java +++ b/server/src/main/java/org/opensearch/search/profile/query/QueryProfileBreakdown.java @@ -34,6 +34,10 @@ import org.opensearch.search.profile.AbstractProfileBreakdown; import org.opensearch.search.profile.ContextualProfileBreakdown; +import org.opensearch.search.profile.ProfileMetric; + +import java.util.Collection; +import java.util.function.Supplier; /** * A record of timings for the various operations that may happen during query execution. @@ -42,15 +46,15 @@ * * @opensearch.internal */ -public final class QueryProfileBreakdown extends ContextualProfileBreakdown { +public final class QueryProfileBreakdown extends ContextualProfileBreakdown { - /** Sole constructor. */ - public QueryProfileBreakdown() { - super(QueryTimingType.class); + public QueryProfileBreakdown(Collection> metrics) { + super(metrics); } @Override - public AbstractProfileBreakdown context(Object context) { + public AbstractProfileBreakdown context(Object context) { return this; } + } diff --git a/server/src/main/java/org/opensearch/search/profile/query/QueryProfiler.java b/server/src/main/java/org/opensearch/search/profile/query/QueryProfiler.java index 78e65c5bfa257..9791f9af2035c 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/QueryProfiler.java +++ b/server/src/main/java/org/opensearch/search/profile/query/QueryProfiler.java @@ -53,7 +53,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class QueryProfiler extends AbstractProfiler, Query> { +public class QueryProfiler extends AbstractProfiler { /** * The root Collector used in the search diff --git a/server/src/test/java/org/opensearch/search/profile/AbstractProfileBreakdownTests.java b/server/src/test/java/org/opensearch/search/profile/AbstractProfileBreakdownTests.java new file mode 100644 index 0000000000000..bfc2092ee7b0f --- /dev/null +++ b/server/src/test/java/org/opensearch/search/profile/AbstractProfileBreakdownTests.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.profile; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +public class AbstractProfileBreakdownTests extends OpenSearchTestCase { + private enum TestType { + STAT_1, + STAT_2; + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + private static class TestProfileBreakdown extends AbstractProfileBreakdown { + Map stats; + + TestProfileBreakdown() { + super(List.of()); + stats = new HashMap(); + long counter = 123; + for (TestType type : TestType.values()) { + stats.put(type.toString(), counter++); + } + } + + @Override + public Map toBreakdownMap() { + return Collections.unmodifiableMap(stats); + } + + @Override + public Map toDebugMap() { + return Map.of("test_debug", 1234L); + } + } + + public void testToBreakdownMap() { + AbstractProfileBreakdown breakdown = new TestProfileBreakdown(); + Map stats = new HashMap<>(); + stats.put("stat_1", 123L); + stats.put("stat_2", 124L); + assertEquals(stats, breakdown.toBreakdownMap()); + assertEquals(Collections.singletonMap("test_debug", 1234L), breakdown.toDebugMap()); + } +} diff --git a/server/src/test/java/org/opensearch/search/profile/ProfileMetricTests.java b/server/src/test/java/org/opensearch/search/profile/ProfileMetricTests.java new file mode 100644 index 0000000000000..453573b97c96a --- /dev/null +++ b/server/src/test/java/org/opensearch/search/profile/ProfileMetricTests.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.profile; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +public class ProfileMetricTests extends OpenSearchTestCase { + + private static class TestMetric extends ProfileMetric { + + private long value = 0L; + + public TestMetric(String name) { + super(name); + } + + public void setValue(long value) { + this.value = value; + } + + @Override + public Map toBreakdownMap() { + return Map.of("test_metric", value); + } + } + + public void testNonTimingMetric() { + TestMetric test_metric = new TestMetric("test_metric"); + test_metric.setValue(1234L); + assertEquals(test_metric.getName(), "test_metric"); + Map map = test_metric.toBreakdownMap(); + assertEquals(map.get("test_metric").longValue(), 1234L); + } + + public static Collection> getNonTimingMetric() { + return List.of(() -> new TestMetric("test_metric")); + } +} diff --git a/server/src/test/java/org/opensearch/search/profile/TimerTests.java b/server/src/test/java/org/opensearch/search/profile/TimerTests.java index 5997292eb8f56..8634522dedcda 100644 --- a/server/src/test/java/org/opensearch/search/profile/TimerTests.java +++ b/server/src/test/java/org/opensearch/search/profile/TimerTests.java @@ -34,13 +34,14 @@ import org.opensearch.test.OpenSearchTestCase; +import java.util.Map; import java.util.concurrent.atomic.AtomicLong; public class TimerTests extends OpenSearchTestCase { public void testTimingInterval() { final AtomicLong nanoTimeCallCounter = new AtomicLong(); - Timer t = new Timer() { + Timer t = new Timer("test") { long time = 50; @Override @@ -63,7 +64,7 @@ long nanoTime() { } public void testExtrapolate() { - Timer t = new Timer() { + Timer t = new Timer("test") { long time = 50; @Override @@ -84,4 +85,16 @@ long nanoTime() { } } + public void testTimerBreakdownMap() { + Timer t = new Timer(123L, 2L, 1234L, 0L, 12345L, "test"); + Map map = t.toBreakdownMap(); + assertEquals(map.size(), 3); + assertEquals(map.get("test").longValue(), 123L); + assertEquals(map.get("test_count").longValue(), 2L); + assertEquals(map.get("test_start_time").longValue(), 12345L); + + Timer t1 = new Timer(123L, 2L, 1234L, 0L, 12345L, "test1"); + assertEquals(t1.getName(), "test1"); + } + } diff --git a/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java b/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java index db14eb90ef839..d3bea3bd4c5ce 100644 --- a/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java +++ b/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java @@ -25,16 +25,22 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Collector; import org.apache.lucene.store.Directory; -import org.opensearch.search.profile.AbstractProfileBreakdown; +import org.opensearch.search.profile.ContextualProfileBreakdown; +import org.opensearch.search.profile.ProfileMetric; +import org.opensearch.search.profile.ProfileMetricTests; +import org.opensearch.search.profile.ProfileMetricUtil; import org.opensearch.search.profile.Timer; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; +import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.function.Supplier; -import static org.opensearch.search.profile.AbstractProfileBreakdown.TIMING_TYPE_COUNT_SUFFIX; -import static org.opensearch.search.profile.AbstractProfileBreakdown.TIMING_TYPE_START_TIME_SUFFIX; +import static org.opensearch.search.profile.Timer.TIMING_TYPE_COUNT_SUFFIX; +import static org.opensearch.search.profile.Timer.TIMING_TYPE_START_TIME_SUFFIX; import static org.opensearch.search.profile.query.ConcurrentQueryProfileBreakdown.MIN_PREFIX; import static org.opensearch.search.profile.query.ConcurrentQueryProfileBreakdown.SLICE_END_TIME_SUFFIX; import static org.opensearch.search.profile.query.ConcurrentQueryProfileBreakdown.SLICE_START_TIME_SUFFIX; @@ -42,11 +48,15 @@ public class ConcurrentQueryProfileBreakdownTests extends OpenSearchTestCase { private ConcurrentQueryProfileBreakdown testQueryProfileBreakdown; + private ConcurrentQueryProfileBreakdown testQueryProfileBreakdownCombined; private Timer createWeightTimer; @Before public void setup() { - testQueryProfileBreakdown = new ConcurrentQueryProfileBreakdown(); + testQueryProfileBreakdown = new ConcurrentQueryProfileBreakdown(ProfileMetricUtil.getDefaultQueryProfileMetrics()); + Collection> combinedMetrics = ProfileMetricUtil.getDefaultQueryProfileMetrics(); + combinedMetrics.addAll(ProfileMetricTests.getNonTimingMetric()); + testQueryProfileBreakdownCombined = new ConcurrentQueryProfileBreakdown(combinedMetrics); createWeightTimer = testQueryProfileBreakdown.getTimer(QueryTimingType.CREATE_WEIGHT); try { createWeightTimer.start(); @@ -105,10 +115,7 @@ public void testBuildSliceLevelBreakdownWithSingleSlice() throws Exception { final long createWeightEarliestStartTime = createWeightTimer.getEarliestTimerStartTime(); final long createWeightEndTime = createWeightEarliestStartTime + createWeightTimer.getApproximateTiming(); final Map leafProfileBreakdownMap = getLeafBreakdownMap(createWeightEndTime + 10, 10, 1); - final AbstractProfileBreakdown leafProfileBreakdown = new TestQueryProfileBreakdown( - QueryTimingType.class, - leafProfileBreakdownMap - ); + final ContextualProfileBreakdown leafProfileBreakdown = new TestQueryProfileBreakdown(leafProfileBreakdownMap); testQueryProfileBreakdown.associateCollectorToLeaves(sliceCollector, sliceLeaf); testQueryProfileBreakdown.getContexts().put(sliceLeaf, leafProfileBreakdown); final Map> sliceBreakdownMap = testQueryProfileBreakdown.buildSliceLevelBreakdown(); @@ -156,14 +163,8 @@ public void testBuildSliceLevelBreakdownWithMultipleSlices() throws Exception { final long createWeightEndTime = createWeightEarliestStartTime + createWeightTimer.getApproximateTiming(); final Map leafProfileBreakdownMap_1 = getLeafBreakdownMap(createWeightEndTime + 10, 10, 1); final Map leafProfileBreakdownMap_2 = getLeafBreakdownMap(createWeightEndTime + 40, 10, 1); - final AbstractProfileBreakdown leafProfileBreakdown_1 = new TestQueryProfileBreakdown( - QueryTimingType.class, - leafProfileBreakdownMap_1 - ); - final AbstractProfileBreakdown leafProfileBreakdown_2 = new TestQueryProfileBreakdown( - QueryTimingType.class, - leafProfileBreakdownMap_2 - ); + final ContextualProfileBreakdown leafProfileBreakdown_1 = new TestQueryProfileBreakdown(leafProfileBreakdownMap_1); + final ContextualProfileBreakdown leafProfileBreakdown_2 = new TestQueryProfileBreakdown(leafProfileBreakdownMap_2); testQueryProfileBreakdown.associateCollectorToLeaves(sliceCollector_1, directoryReader.leaves().get(0)); testQueryProfileBreakdown.associateCollectorToLeaves(sliceCollector_2, directoryReader.leaves().get(1)); testQueryProfileBreakdown.getContexts().put(directoryReader.leaves().get(0), leafProfileBreakdown_1); @@ -222,14 +223,8 @@ public void testBreakDownMapWithMultipleSlices() throws Exception { final long createWeightEndTime = createWeightEarliestStartTime + createWeightTimer.getApproximateTiming(); final Map leafProfileBreakdownMap_1 = getLeafBreakdownMap(createWeightEndTime + 10, 10, 1); final Map leafProfileBreakdownMap_2 = getLeafBreakdownMap(createWeightEndTime + 40, 20, 1); - final AbstractProfileBreakdown leafProfileBreakdown_1 = new TestQueryProfileBreakdown( - QueryTimingType.class, - leafProfileBreakdownMap_1 - ); - final AbstractProfileBreakdown leafProfileBreakdown_2 = new TestQueryProfileBreakdown( - QueryTimingType.class, - leafProfileBreakdownMap_2 - ); + final ContextualProfileBreakdown leafProfileBreakdown_1 = new TestQueryProfileBreakdown(leafProfileBreakdownMap_1); + final ContextualProfileBreakdown leafProfileBreakdown_2 = new TestQueryProfileBreakdown(leafProfileBreakdownMap_2); testQueryProfileBreakdown.associateCollectorToLeaves(sliceCollector_1, directoryReader.leaves().get(0)); testQueryProfileBreakdown.associateCollectorToLeaves(sliceCollector_2, directoryReader.leaves().get(1)); testQueryProfileBreakdown.getContexts().put(directoryReader.leaves().get(0), leafProfileBreakdown_1); @@ -283,10 +278,7 @@ public void testBreakDownMapWithMultipleSlicesAndOneSliceWithNoLeafContext() thr final long createWeightEarliestStartTime = createWeightTimer.getEarliestTimerStartTime(); final long createWeightEndTime = createWeightEarliestStartTime + createWeightTimer.getApproximateTiming(); final Map leafProfileBreakdownMap_1 = getLeafBreakdownMap(createWeightEndTime + 10, 10, 1); - final AbstractProfileBreakdown leafProfileBreakdown_1 = new TestQueryProfileBreakdown( - QueryTimingType.class, - leafProfileBreakdownMap_1 - ); + final ContextualProfileBreakdown leafProfileBreakdown_1 = new TestQueryProfileBreakdown(leafProfileBreakdownMap_1); testQueryProfileBreakdown.associateCollectorToLeaves(sliceCollector_1, directoryReader.leaves().get(0)); testQueryProfileBreakdown.associateCollectorToLeaves(sliceCollector_2, directoryReader.leaves().get(1)); testQueryProfileBreakdown.getContexts().put(directoryReader.leaves().get(0), leafProfileBreakdown_1); @@ -339,10 +331,7 @@ public void testOneLeafContextWithEmptySliceCollectorsToLeaves() throws Exceptio final long createWeightEarliestStartTime = createWeightTimer.getEarliestTimerStartTime(); final long createWeightEndTime = createWeightEarliestStartTime + createWeightTimer.getApproximateTiming(); final Map leafProfileBreakdownMap_1 = getLeafBreakdownMap(createWeightEndTime + 10, 10, 1); - final AbstractProfileBreakdown leafProfileBreakdown_1 = new TestQueryProfileBreakdown( - QueryTimingType.class, - leafProfileBreakdownMap_1 - ); + final ContextualProfileBreakdown leafProfileBreakdown_1 = new TestQueryProfileBreakdown(leafProfileBreakdownMap_1); testQueryProfileBreakdown.getContexts().put(directoryReader.leaves().get(0), leafProfileBreakdown_1); final Map queryBreakDownMap = testQueryProfileBreakdown.toBreakdownMap(); assertFalse(queryBreakDownMap == null || queryBreakDownMap.isEmpty()); @@ -385,6 +374,54 @@ public void testOneLeafContextWithEmptySliceCollectorsToLeaves() throws Exceptio directory.close(); } + public void testBuildSliceLevelBreakdownWithSingleSliceCombinedMetricOnly() throws Exception { + final DirectoryReader directoryReader = getDirectoryReader(1); + final Directory directory = directoryReader.directory(); + final LeafReaderContext sliceLeaf = directoryReader.leaves().get(0); + final Collector sliceCollector = mock(Collector.class); + final long createWeightEarliestStartTime = createWeightTimer.getEarliestTimerStartTime(); + final long createWeightEndTime = createWeightEarliestStartTime + createWeightTimer.getApproximateTiming(); + final Map leafProfileBreakdownMap = getCombinedLeafBreakdownMap(createWeightEndTime + 10, 10, 1); + final ContextualProfileBreakdown leafProfileBreakdown = new TestQueryProfileBreakdown(leafProfileBreakdownMap); + testQueryProfileBreakdownCombined.associateCollectorToLeaves(sliceCollector, sliceLeaf); + testQueryProfileBreakdownCombined.getContexts().put(sliceLeaf, leafProfileBreakdown); + final Map> sliceBreakdownMap = testQueryProfileBreakdownCombined.buildSliceLevelBreakdown(); + assertFalse(sliceBreakdownMap == null || sliceBreakdownMap.isEmpty()); + assertEquals(1, sliceBreakdownMap.size()); + assertTrue(sliceBreakdownMap.containsKey(sliceCollector)); + + final Map sliceBreakdown = sliceBreakdownMap.entrySet().iterator().next().getValue(); + for (QueryTimingType timingType : QueryTimingType.values()) { + String timingTypeKey = timingType.toString(); + String timingTypeCountKey = timingTypeKey + TIMING_TYPE_COUNT_SUFFIX; + + if (timingType.equals(QueryTimingType.CREATE_WEIGHT)) { + // there should be no entry for create weight at slice level breakdown map + assertNull(sliceBreakdown.get(timingTypeKey)); + assertNull(sliceBreakdown.get(timingTypeCountKey)); + continue; + } + + // for other timing type we will have all the value and will be same as leaf breakdown as there is single slice and single leaf + assertEquals(leafProfileBreakdownMap.get(timingTypeKey), sliceBreakdown.get(timingTypeKey)); + assertEquals(leafProfileBreakdownMap.get(timingTypeCountKey), sliceBreakdown.get(timingTypeCountKey)); + assertEquals( + leafProfileBreakdownMap.get(timingTypeKey + TIMING_TYPE_START_TIME_SUFFIX), + sliceBreakdown.get(timingTypeKey + SLICE_START_TIME_SUFFIX) + ); + assertEquals( + leafProfileBreakdownMap.get(timingTypeKey + TIMING_TYPE_START_TIME_SUFFIX) + leafProfileBreakdownMap.get(timingTypeKey), + (long) sliceBreakdown.get(timingTypeKey + SLICE_END_TIME_SUFFIX) + ); + } + assertEquals(leafProfileBreakdownMap.get("value"), sliceBreakdown.get("value")); + assertEquals(10, testQueryProfileBreakdownCombined.getMaxSliceNodeTime()); + assertEquals(10, testQueryProfileBreakdownCombined.getMinSliceNodeTime()); + assertEquals(10, testQueryProfileBreakdownCombined.getAvgSliceNodeTime()); + directoryReader.close(); + directory.close(); + } + private Map getLeafBreakdownMap(long startTime, long timeTaken, long count) { Map leafBreakDownMap = new HashMap<>(); for (QueryTimingType timingType : QueryTimingType.values()) { @@ -400,6 +437,22 @@ private Map getLeafBreakdownMap(long startTime, long timeTaken, lo return leafBreakDownMap; } + private Map getCombinedLeafBreakdownMap(long startTime, long timeTaken, long count) { + Map leafBreakDownMap = new HashMap<>(); + for (QueryTimingType timingType : QueryTimingType.values()) { + if (timingType.equals(QueryTimingType.CREATE_WEIGHT)) { + // don't add anything + continue; + } + String timingTypeKey = timingType.toString(); + leafBreakDownMap.put(timingTypeKey, timeTaken); + leafBreakDownMap.put(timingTypeKey + TIMING_TYPE_COUNT_SUFFIX, count); + leafBreakDownMap.put(timingTypeKey + TIMING_TYPE_START_TIME_SUFFIX, startTime); + } + leafBreakDownMap.put("test_metric", 123L); + return leafBreakDownMap; + } + private DirectoryReader getDirectoryReader(int numLeaves) throws Exception { final Directory directory = newDirectory(); IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(NoMergePolicy.INSTANCE)); @@ -416,11 +469,11 @@ private DirectoryReader getDirectoryReader(int numLeaves) throws Exception { return DirectoryReader.open(directory); } - private static class TestQueryProfileBreakdown extends AbstractProfileBreakdown { + private static class TestQueryProfileBreakdown extends ContextualProfileBreakdown { private Map breakdownMap; - public TestQueryProfileBreakdown(Class clazz, Map breakdownMap) { - super(clazz); + public TestQueryProfileBreakdown(Map breakdownMap) { + super(List.of()); this.breakdownMap = breakdownMap; } @@ -428,5 +481,10 @@ public TestQueryProfileBreakdown(Class clazz, Map public Map toBreakdownMap() { return breakdownMap; } + + @Override + public ContextualProfileBreakdown context(Object context) { + return null; + } } } diff --git a/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfilerTests.java b/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfilerTests.java index 736bbcdd9e8dd..2a40c7608feaa 100644 --- a/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfilerTests.java +++ b/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfilerTests.java @@ -21,9 +21,9 @@ public class ConcurrentQueryProfilerTests extends OpenSearchTestCase { public void testMergeRewriteTimeIntervals() { ConcurrentQueryProfiler profiler = new ConcurrentQueryProfiler(new ConcurrentQueryProfileTree()); List timers = new LinkedList<>(); - timers.add(new Timer(217134L, 1L, 1L, 0L, 553074511206907L)); - timers.add(new Timer(228954L, 1L, 1L, 0L, 553074509287335L)); - timers.add(new Timer(228954L, 1L, 1L, 0L, 553074509287336L)); + timers.add(new Timer(217134L, 1L, 1L, 0L, 553074511206907L, "test1")); + timers.add(new Timer(228954L, 1L, 1L, 0L, 553074509287335L, "test2")); + timers.add(new Timer(228954L, 1L, 1L, 0L, 553074509287336L, "test3")); LinkedList mergedIntervals = profiler.mergeRewriteTimeIntervals(timers); assertThat(mergedIntervals.size(), equalTo(2)); long[] interval = mergedIntervals.get(0); diff --git a/server/src/test/java/org/opensearch/search/profile/query/ProfileScorerTests.java b/server/src/test/java/org/opensearch/search/profile/query/ProfileScorerTests.java index a4ff92ec556e9..0112fa0839965 100644 --- a/server/src/test/java/org/opensearch/search/profile/query/ProfileScorerTests.java +++ b/server/src/test/java/org/opensearch/search/profile/query/ProfileScorerTests.java @@ -40,6 +40,7 @@ import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; +import org.opensearch.search.profile.ProfileMetricUtil; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -84,7 +85,7 @@ public void testPropagateMinCompetitiveScore() throws IOException { Query query = new MatchAllDocsQuery(); Weight weight = query.createWeight(new IndexSearcher(new MultiReader()), ScoreMode.TOP_SCORES, 1f); FakeScorer fakeScorer = new FakeScorer(weight); - QueryProfileBreakdown profile = new QueryProfileBreakdown(); + QueryProfileBreakdown profile = new QueryProfileBreakdown(ProfileMetricUtil.getDefaultQueryProfileMetrics()); ProfileScorer profileScorer = new ProfileScorer(fakeScorer, profile); profileScorer.setMinCompetitiveScore(0.42f); assertEquals(0.42f, fakeScorer.minCompetitiveScore, 0f); @@ -94,7 +95,7 @@ public void testPropagateMaxScore() throws IOException { Query query = new MatchAllDocsQuery(); Weight weight = query.createWeight(new IndexSearcher(new MultiReader()), ScoreMode.TOP_SCORES, 1f); FakeScorer fakeScorer = new FakeScorer(weight); - QueryProfileBreakdown profile = new QueryProfileBreakdown(); + QueryProfileBreakdown profile = new QueryProfileBreakdown(ProfileMetricUtil.getDefaultQueryProfileMetrics()); ProfileScorer profileScorer = new ProfileScorer(fakeScorer, profile); profileScorer.setMinCompetitiveScore(0.42f); fakeScorer.maxScore = 42f; From 1af55d2f836ddc6b7abf425de1b4f72211888fed Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Tue, 1 Jul 2025 07:40:02 -0700 Subject: [PATCH 09/51] [Bug Fix] Fix the backward compatibility regression with `COMPLEMENT` for Regexp queries introduced in OpenSearch 3.0 (#18640) * FIx ~ bug Signed-off-by: Prudhvi Godithi * Update changelog Signed-off-by: Prudhvi Godithi --------- Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 1 + .../search/simple/SimpleSearchIT.java | 19 ++++++++++ .../index/query/RegexpQueryBuilder.java | 19 +++++++++- .../index/query/RegexpQueryBuilderTests.java | 38 ++++++++++++++++++- 4 files changed, 75 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcfb8e3195134..7a548cccac6f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Autotagging] Fix delete rule event consumption in InMemoryRuleProcessingService ([#18628](https://github.com/opensearch-project/OpenSearch/pull/18628)) - Cannot communicate with HTTP/2 when reactor-netty is enabled ([#18599](https://github.com/opensearch-project/OpenSearch/pull/18599)) - Fix the visit of sub queries for HasParentQuery and HasChildQuery ([#18621](https://github.com/opensearch-project/OpenSearch/pull/18621)) +- Fix the backward compatibility regression with COMPLEMENT for Regexp queries introduced in OpenSearch 3.0 ([#18640](https://github.com/opensearch-project/OpenSearch/pull/18640)) ### Security diff --git a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java index 419ea2f74019c..d32bad5f17d2a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java @@ -49,6 +49,8 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.ConstantScoreQueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.RegexpFlag; +import org.opensearch.index.query.RegexpQueryBuilder; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.search.rescore.QueryRescorerBuilder; import org.opensearch.search.sort.SortOrder; @@ -59,7 +61,9 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; @@ -763,4 +767,19 @@ private void assertRescoreWindowFails(int windowSize) { ) ); } + + public void testRegexQueryWithComplementFlag() { + createIndex("test_regex"); + client().prepareIndex("test_regex").setId("1").setSource("text", "abc").get(); + client().prepareIndex("test_regex").setId("2").setSource("text", "adc").get(); + client().prepareIndex("test_regex").setId("3").setSource("text", "acc").get(); + refresh(); + RegexpQueryBuilder query = new RegexpQueryBuilder("text.keyword", "a~bc"); + query.flags(RegexpFlag.COMPLEMENT); + SearchResponse response = client().prepareSearch("test_regex").setQuery(query).get(); + assertEquals("COMPLEMENT should match 2 documents", 2L, response.getHits().getTotalHits().value()); + Set matchedIds = Arrays.stream(response.getHits().getHits()).map(hit -> hit.getId()).collect(Collectors.toSet()); + assertEquals("Should match exactly 2 documents", 2, matchedIds.size()); + assertTrue("Should match documents 2 and 3", matchedIds.containsAll(Arrays.asList("2", "3"))); + } } diff --git a/server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java index f0da4d5736c0f..692ce355d003e 100644 --- a/server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java @@ -38,6 +38,7 @@ import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; +import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.ParseField; @@ -60,6 +61,9 @@ * @opensearch.internal */ public class RegexpQueryBuilder extends AbstractQueryBuilder implements MultiTermQueryBuilder { + + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RegexpQueryBuilder.class); + public static final String NAME = "regexp"; public static final int DEFAULT_FLAGS_VALUE = RegexpFlag.ALL.value(); @@ -294,12 +298,25 @@ protected Query doToQuery(QueryShardContext context) throws QueryShardException, + "] index level setting." ); } + + // Check if COMPLEMENT flag is being used + // The COMPLEMENT flag maps to Lucene's DEPRECATED_COMPLEMENT which is marked for removal in Lucene 11 + // This deprecation warning helps users migrate their queries before the feature is completely removed + if ((syntaxFlagsValue & RegexpFlag.COMPLEMENT.value()) != 0) { + deprecationLogger.deprecate( + "regexp_complement_operator", + "The complement operator (~) for arbitrary patterns in regexp queries is deprecated and will be removed in a future version. " + + "Consider rewriting your query to use character class negation [^...] or other query types." + ); + } + MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(rewrite, null, LoggingDeprecationHandler.INSTANCE); int matchFlagsValue = caseInsensitive ? RegExp.ASCII_CASE_INSENSITIVE : 0; Query query = null; // For BWC we mask irrelevant bits (RegExp changed ALL from 0xffff to 0xff) - int sanitisedSyntaxFlag = syntaxFlagsValue & RegExp.ALL; + // The hexadecimal for DEPRECATED_COMPLEMENT is 0x10000. The OR condition ensures COMPLEMENT ~ is preserved + int sanitisedSyntaxFlag = syntaxFlagsValue & (RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT); MappedFieldType fieldType = context.fieldMapper(fieldName); if (fieldType != null) { diff --git a/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java index 30c8c07fa2c27..7f840cb789b46 100644 --- a/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java @@ -55,7 +55,15 @@ protected RegexpQueryBuilder doCreateTestQueryBuilder() { List flags = new ArrayList<>(); int iter = randomInt(5); for (int i = 0; i < iter; i++) { - flags.add(randomFrom(RegexpFlag.values())); + // Exclude COMPLEMENT from random selection to avoid deprecation warnings + RegexpFlag[] availableFlags = { + RegexpFlag.INTERSECTION, + RegexpFlag.EMPTY, + RegexpFlag.ANYSTRING, + RegexpFlag.INTERVAL, + RegexpFlag.NONE, + RegexpFlag.ALL }; + flags.add(randomFrom(availableFlags)); } query.flags(flags.toArray(new RegexpFlag[0])); } @@ -162,4 +170,32 @@ public void testParseFailsWithMultipleFields() throws IOException { e = expectThrows(ParsingException.class, () -> parseQuery(shortJson)); assertEquals("[regexp] query doesn't support multiple fields, found [user1] and [user2]", e.getMessage()); } + + // Test that COMPLEMENT flag triggers deprecation warning + public void testComplementFlagDeprecation() throws IOException { + RegexpQueryBuilder query = new RegexpQueryBuilder("field", "a~bc"); + query.flags(RegexpFlag.COMPLEMENT); + QueryShardContext context = createShardContext(); + Query luceneQuery = query.toQuery(context); + assertNotNull(luceneQuery); + assertThat(luceneQuery, instanceOf(RegexpQuery.class)); + assertWarnings( + "The complement operator (~) for arbitrary patterns in regexp queries is deprecated " + + "and will be removed in a future version. Consider rewriting your query to use character class negation [^...] or other query types." + ); + } + + // Separate test for COMPLEMENT flag Cacheability + public void testComplementFlagCacheability() throws IOException { + RegexpQueryBuilder queryBuilder = new RegexpQueryBuilder("field", "pattern"); + queryBuilder.flags(RegexpFlag.COMPLEMENT); + QueryShardContext context = createShardContext(); + QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new QueryShardContext(context)); + assertNotNull(rewriteQuery.toQuery(context)); + assertTrue("query should be cacheable: " + queryBuilder, context.isCacheable()); + assertWarnings( + "The complement operator (~) for arbitrary patterns in regexp queries is deprecated " + + "and will be removed in a future version. Consider rewriting your query to use character class negation [^...] or other query types." + ); + } } From 223f9fd85077b99c9297d39f65d94f9c29624808 Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Tue, 1 Jul 2025 09:57:46 -0700 Subject: [PATCH 10/51] [WLM] Make resource usage value cacheable for NodeDuressTrackers (#18649) --------- Signed-off-by: Kaushal Kumar Signed-off-by: Ankit Jain Co-authored-by: Ankit Jain --- CHANGELOG.md | 1 + .../common/util/TimeBasedExpiryTracker.java | 45 ++++++++++++++++++ .../trackers/NodeDuressTrackers.java | 22 ++++++++- .../TimeBasedCacheExpiryTrackerTests.java | 47 +++++++++++++++++++ .../SearchBackpressureServiceTests.java | 20 ++++---- .../trackers/NodeDuressTrackersTests.java | 19 ++++---- 6 files changed, 136 insertions(+), 18 deletions(-) create mode 100644 server/src/main/java/org/opensearch/common/util/TimeBasedExpiryTracker.java create mode 100644 server/src/test/java/org/opensearch/common/util/TimeBasedCacheExpiryTrackerTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a548cccac6f9..78e3f703c84e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Changed - Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570)) - Update SecureAuxTransportSettingsProvider to distinguish between aux transport types ([#18616](https://github.com/opensearch-project/OpenSearch/pull/18616)) +- Make node duress values cacheable ([#18649](https://github.com/opensearch-project/OpenSearch/pull/18649)) - Making multi rate limiters in repository dynamic [#18069](https://github.com/opensearch-project/OpenSearch/pull/18069) ### Dependencies diff --git a/server/src/main/java/org/opensearch/common/util/TimeBasedExpiryTracker.java b/server/src/main/java/org/opensearch/common/util/TimeBasedExpiryTracker.java new file mode 100644 index 0000000000000..c99d0f5be0fa9 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/util/TimeBasedExpiryTracker.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.util; + +import java.util.function.BooleanSupplier; +import java.util.function.LongSupplier; + +/** + * This class can be utilised to track the time based expiration events. + * Clients should be more cautious with the expiry time as this class is not completely thread safe. This is intentional + * as nanoSecond granularity level error for `lastAccessTimeInNanos` is tolerable and can be ignored. + * @opensearch.internal + */ +public class TimeBasedExpiryTracker implements BooleanSupplier { + private final LongSupplier nanoTimeSupplier; + private volatile long lastAccessTimeInNanos; + private final long expiryTimeInNanos; + private static final long ONE_SEC = 1000_000_000; + + public TimeBasedExpiryTracker(LongSupplier nanoTimeSupplier) { + this(nanoTimeSupplier, ONE_SEC); + } + + public TimeBasedExpiryTracker(LongSupplier nanoTimeSupplier, long expiryTimeInNanos) { + this.nanoTimeSupplier = nanoTimeSupplier; + this.lastAccessTimeInNanos = nanoTimeSupplier.getAsLong(); + this.expiryTimeInNanos = expiryTimeInNanos; + } + + @Override + public boolean getAsBoolean() { + final long currentTime = nanoTimeSupplier.getAsLong(); + final boolean isExpired = (currentTime - lastAccessTimeInNanos) > expiryTimeInNanos; + if (isExpired) { + lastAccessTimeInNanos = currentTime; + } + return isExpired; + } +} diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java index c27c50ac12c0f..2cf5f63144e9a 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java +++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java @@ -9,9 +9,11 @@ package org.opensearch.search.backpressure.trackers; import org.opensearch.common.util.Streak; +import org.opensearch.common.util.TimeBasedExpiryTracker; import org.opensearch.wlm.ResourceType; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.BooleanSupplier; import java.util.function.IntSupplier; @@ -22,9 +24,19 @@ */ public class NodeDuressTrackers { private final Map duressTrackers; + private final Map resourceDuressCache = new ConcurrentHashMap<>(); + private final BooleanSupplier nodeDuressCacheExpiryChecker; public NodeDuressTrackers(Map duressTrackers) { + this(duressTrackers, new TimeBasedExpiryTracker(System::nanoTime)); + } + + public NodeDuressTrackers(Map duressTrackers, BooleanSupplier nodeDuressCacheExpiryChecker) { this.duressTrackers = duressTrackers; + for (ResourceType resourceType : ResourceType.values()) { + resourceDuressCache.put(resourceType, false); + } + this.nodeDuressCacheExpiryChecker = nodeDuressCacheExpiryChecker; } /** @@ -32,7 +44,8 @@ public NodeDuressTrackers(Map duressTrackers) { * @return Boolean */ public boolean isResourceInDuress(ResourceType resourceType) { - return duressTrackers.get(resourceType).test(); + updateCache(); + return resourceDuressCache.get(resourceType); } /** @@ -48,6 +61,13 @@ public boolean isNodeInDuress() { return false; } + private void updateCache() { + if (nodeDuressCacheExpiryChecker.getAsBoolean()) { + for (ResourceType resourceType : ResourceType.values()) + resourceDuressCache.put(resourceType, duressTrackers.get(resourceType).test()); + } + } + /** * NodeDuressTracker is used to check if the node is in duress * @opensearch.internal diff --git a/server/src/test/java/org/opensearch/common/util/TimeBasedCacheExpiryTrackerTests.java b/server/src/test/java/org/opensearch/common/util/TimeBasedCacheExpiryTrackerTests.java new file mode 100644 index 0000000000000..c3c6ad275c7f1 --- /dev/null +++ b/server/src/test/java/org/opensearch/common/util/TimeBasedCacheExpiryTrackerTests.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.util; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.function.LongSupplier; + +public class TimeBasedCacheExpiryTrackerTests extends OpenSearchTestCase { + TimeBasedExpiryTracker sut; + final long ONE_SEC = 1_000_000_000; + + public void testExpiryEvent() { + TestTimeSupplier testTimeSupplier = new TestTimeSupplier(); + sut = new TimeBasedExpiryTracker(testTimeSupplier); + + testTimeSupplier.advanceClockBy(2 * ONE_SEC); + assertTrue(sut.getAsBoolean()); + } + + public void testNonExpiryEvent() { + TestTimeSupplier testTimeSupplier = new TestTimeSupplier(); + sut = new TimeBasedExpiryTracker(testTimeSupplier); + + testTimeSupplier.advanceClockBy(ONE_SEC / 2); + assertFalse(sut.getAsBoolean()); + } + + public static class TestTimeSupplier implements LongSupplier { + long currentTime = System.nanoTime(); + + @Override + public long getAsLong() { + return currentTime; + } + + public void advanceClockBy(long nanos) { + currentTime += nanos; + } + } +} diff --git a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java index 8cb22201da1b6..8e604824b73a6 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java @@ -56,6 +56,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; import java.util.function.LongSupplier; import static org.opensearch.search.backpressure.SearchBackpressureTestHelpers.createMockTaskWithResourceStats; @@ -74,6 +75,7 @@ import static org.mockito.Mockito.when; public class SearchBackpressureServiceTests extends OpenSearchTestCase { + final BooleanSupplier resourceCacheExpiryChecker = () -> true; MockTransportService transportService; TaskManager taskManager; ThreadPool threadPool; @@ -101,8 +103,8 @@ public void testIsNodeInDuress() { AtomicReference cpuUsage = new AtomicReference<>(); AtomicReference heapUsage = new AtomicReference<>(); - NodeDuressTracker cpuUsageTracker = new NodeDuressTracker(() -> cpuUsage.get() >= 0.5, () -> 3); - NodeDuressTracker heapUsageTracker = new NodeDuressTracker(() -> heapUsage.get() >= 0.5, () -> 3); + NodeDuressTracker cpuUsageTracker = new NodeDuressTracker(() -> cpuUsage.get() >= 0.5, () -> 5); + NodeDuressTracker heapUsageTracker = new NodeDuressTracker(() -> heapUsage.get() >= 0.5, () -> 6); EnumMap duressTrackers = new EnumMap<>(ResourceType.class) { { @@ -121,7 +123,7 @@ public void testIsNodeInDuress() { mockTaskResourceTrackingService, threadPool, System::nanoTime, - new NodeDuressTrackers(duressTrackers), + new NodeDuressTrackers(duressTrackers, resourceCacheExpiryChecker), new TaskResourceUsageTrackers(), new TaskResourceUsageTrackers(), taskManager, @@ -167,7 +169,7 @@ public void testTrackerStateUpdateOnSearchTaskCompletion() { mockTaskResourceTrackingService, threadPool, mockTimeNanosSupplier, - new NodeDuressTrackers(new EnumMap<>(ResourceType.class)), + new NodeDuressTrackers(new EnumMap<>(ResourceType.class), resourceCacheExpiryChecker), taskResourceUsageTrackers, new TaskResourceUsageTrackers(), taskManager, @@ -201,7 +203,7 @@ public void testTrackerStateUpdateOnSearchShardTaskCompletion() { mockTaskResourceTrackingService, threadPool, mockTimeNanosSupplier, - new NodeDuressTrackers(new EnumMap<>(ResourceType.class)), + new NodeDuressTrackers(new EnumMap<>(ResourceType.class), resourceCacheExpiryChecker), new TaskResourceUsageTrackers(), taskResourceUsageTrackers, taskManager, @@ -256,7 +258,7 @@ public void testSearchTaskInFlightCancellation() { mockTaskResourceTrackingService, threadPool, mockTimeNanosSupplier, - new NodeDuressTrackers(duressTrackers), + new NodeDuressTrackers(duressTrackers, resourceCacheExpiryChecker), taskResourceUsageTrackers, new TaskResourceUsageTrackers(), mockTaskManager, @@ -329,7 +331,7 @@ public void testSearchShardTaskInFlightCancellation() { put(CPU, mockNodeDuressTracker); } }; - NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(duressTrackers); + NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(duressTrackers, resourceCacheExpiryChecker); TaskResourceUsageTracker mockTaskResourceUsageTracker = getMockedTaskResourceUsageTracker( TaskResourceUsageTrackerType.CPU_USAGE_TRACKER, @@ -427,7 +429,7 @@ public void testNonCancellationOfHeapBasedTasksWhenHeapNotInDuress() { } }; - NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(duressTrackers); + NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(duressTrackers, resourceCacheExpiryChecker); // Creating heap and cpu usage trackers where heap tracker will always evaluate with reasons to cancel the // tasks but heap based cancellation should not happen because heap is not in duress @@ -525,7 +527,7 @@ public void testNonCancellationWhenSearchTrafficIsNotQualifyingForCancellation() } }; - NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(duressTrackers); + NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(duressTrackers, resourceCacheExpiryChecker); // Creating heap and cpu usage trackers where heap tracker will always evaluate with reasons to cancel the // tasks but heap based cancellation should not happen because heap is not in duress diff --git a/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java b/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java index 7c52840c099d4..f46d84e1034a2 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java @@ -13,9 +13,12 @@ import org.opensearch.wlm.ResourceType; import java.util.EnumMap; +import java.util.function.BooleanSupplier; public class NodeDuressTrackersTests extends OpenSearchTestCase { + final BooleanSupplier resourceCacheExpiryChecker = () -> true; + public void testNodeNotInDuress() { EnumMap map = new EnumMap<>(ResourceType.class) { { @@ -24,7 +27,7 @@ public void testNodeNotInDuress() { } }; - NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map); + NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map, resourceCacheExpiryChecker); assertFalse(nodeDuressTrackers.isNodeInDuress()); assertFalse(nodeDuressTrackers.isNodeInDuress()); @@ -34,12 +37,12 @@ public void testNodeNotInDuress() { public void testNodeInDuressWhenHeapInDuress() { EnumMap map = new EnumMap<>(ResourceType.class) { { - put(ResourceType.MEMORY, new NodeDuressTracker(() -> true, () -> 3)); + put(ResourceType.MEMORY, new NodeDuressTracker(() -> true, () -> 6)); put(ResourceType.CPU, new NodeDuressTracker(() -> false, () -> 1)); } }; - NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map); + NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map, resourceCacheExpiryChecker); assertFalse(nodeDuressTrackers.isNodeInDuress()); assertFalse(nodeDuressTrackers.isNodeInDuress()); @@ -52,11 +55,11 @@ public void testNodeInDuressWhenCPUInDuress() { EnumMap map = new EnumMap<>(ResourceType.class) { { put(ResourceType.MEMORY, new NodeDuressTracker(() -> false, () -> 1)); - put(ResourceType.CPU, new NodeDuressTracker(() -> true, () -> 3)); + put(ResourceType.CPU, new NodeDuressTracker(() -> true, () -> 5)); } }; - NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map); + NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map, resourceCacheExpiryChecker); assertFalse(nodeDuressTrackers.isNodeInDuress()); assertFalse(nodeDuressTrackers.isNodeInDuress()); @@ -68,12 +71,12 @@ public void testNodeInDuressWhenCPUInDuress() { public void testNodeInDuressWhenCPUAndHeapInDuress() { EnumMap map = new EnumMap<>(ResourceType.class) { { - put(ResourceType.MEMORY, new NodeDuressTracker(() -> true, () -> 3)); - put(ResourceType.CPU, new NodeDuressTracker(() -> false, () -> 3)); + put(ResourceType.MEMORY, new NodeDuressTracker(() -> true, () -> 6)); + put(ResourceType.CPU, new NodeDuressTracker(() -> true, () -> 5)); } }; - NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map); + NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map, resourceCacheExpiryChecker); assertFalse(nodeDuressTrackers.isNodeInDuress()); assertFalse(nodeDuressTrackers.isNodeInDuress()); From 4eb6336363977a563c3afed245876159b3f5b3f7 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 1 Jul 2025 12:36:24 -0700 Subject: [PATCH 11/51] Fix bugs in replication lag computation (#18602) * Fix bug in replication lag computation. This change fixes a bug with replication lag computation to correctly use epoch reference point with Instant.now() and DateUtils. This change also fixes pruning logic to correctly remove the latest synced to checkpoint from tracking. Previously we would only prune up to the latest. This ensures that when a new checkpoint is eventually received we aren't incorrectly computing lag from the synced-to checkpoint. Signed-off-by: Marc Handalian * add changelog entry Signed-off-by: Marc Handalian * Fix flaky test to hardcode time lag between checkpoint arrival. Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- CHANGELOG.md | 1 + .../replication/SegmentReplicator.java | 11 +++++----- .../checkpoint/ReplicationCheckpoint.java | 8 +++++--- .../replication/SegmentReplicatorTests.java | 20 +++++++++++++++---- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78e3f703c84e4..95bca206dd903 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Cannot communicate with HTTP/2 when reactor-netty is enabled ([#18599](https://github.com/opensearch-project/OpenSearch/pull/18599)) - Fix the visit of sub queries for HasParentQuery and HasChildQuery ([#18621](https://github.com/opensearch-project/OpenSearch/pull/18621)) - Fix the backward compatibility regression with COMPLEMENT for Regexp queries introduced in OpenSearch 3.0 ([#18640](https://github.com/opensearch-project/OpenSearch/pull/18640)) +- Fix Replication lag computation ([#18602](https://github.com/opensearch-project/OpenSearch/pull/18602)) ### Security diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicator.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicator.java index d1683f57315f4..72abd4f33e465 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicator.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicator.java @@ -15,6 +15,7 @@ import org.opensearch.OpenSearchCorruptionException; import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; +import org.opensearch.common.time.DateUtils; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.ConcurrentCollections; @@ -31,12 +32,13 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.time.Duration; +import java.time.Instant; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.TimeUnit; import reactor.util.annotation.NonNull; @@ -54,7 +56,7 @@ public class SegmentReplicator { private final ReplicationCollection onGoingReplications; private final ReplicationCollection onGoingMergedSegmentReplications; private final Map completedReplications = ConcurrentCollections.newConcurrentMap(); - private final ConcurrentMap> replicationCheckpointStats = + protected final ConcurrentMap> replicationCheckpointStats = ConcurrentCollections.newConcurrentMap(); private final ConcurrentMap primaryCheckpoint = ConcurrentCollections.newConcurrentMap(); @@ -167,9 +169,8 @@ public ReplicationStats getSegmentReplicationStats(final ShardId shardId) { long bytesBehind = highestEntry.getValue().getBytesBehind(); long replicationLag = bytesBehind > 0L - ? TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lowestEntry.getValue().getTimestamp()) + ? Duration.ofNanos(DateUtils.toLong(Instant.now()) - lowestEntry.getValue().getTimestamp()).toMillis() : 0; - return new ReplicationStats(bytesBehind, bytesBehind, replicationLag); } @@ -217,7 +218,7 @@ protected void pruneCheckpointsUpToLastSync(final IndexShard indexShard) { ); if (existingCheckpointStats != null && !existingCheckpointStats.isEmpty()) { - existingCheckpointStats.keySet().removeIf(key -> key < segmentInfoVersion); + existingCheckpointStats.keySet().removeIf(key -> key <= segmentInfoVersion); Map.Entry lastEntry = existingCheckpointStats.lastEntry(); if (lastEntry != null) { lastEntry.getValue().setBytesBehind(calculateBytesBehind(latestCheckpoint, indexReplicationCheckPoint)); diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java index 8380187a288ba..39c2039191def 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java @@ -11,6 +11,7 @@ import org.opensearch.Version; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.time.DateUtils; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -19,6 +20,7 @@ import org.opensearch.index.store.StoreFileMetadata; import java.io.IOException; +import java.time.Instant; import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -56,11 +58,11 @@ private ReplicationCheckpoint(ShardId shardId, String codec) { length = 0L; this.codec = codec; this.metadataMap = Collections.emptyMap(); - this.createdTimeStamp = System.nanoTime(); + this.createdTimeStamp = DateUtils.toLong(Instant.now()); } public ReplicationCheckpoint(ShardId shardId, long primaryTerm, long segmentsGen, long segmentInfosVersion, String codec) { - this(shardId, primaryTerm, segmentsGen, segmentInfosVersion, 0L, codec, Collections.emptyMap(), System.nanoTime()); + this(shardId, primaryTerm, segmentsGen, segmentInfosVersion, 0L, codec, Collections.emptyMap(), DateUtils.toLong(Instant.now())); } public ReplicationCheckpoint( @@ -79,7 +81,7 @@ public ReplicationCheckpoint( this.length = length; this.codec = codec; this.metadataMap = metadataMap; - this.createdTimeStamp = System.nanoTime(); + this.createdTimeStamp = DateUtils.toLong(Instant.now()); } public ReplicationCheckpoint( diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicatorTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicatorTests.java index 38f1c59bd5b68..a487a0f9a6032 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicatorTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicatorTests.java @@ -21,6 +21,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.settings.Settings; +import org.opensearch.common.time.DateUtils; import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.ReplicationStats; @@ -38,6 +39,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -215,10 +217,10 @@ public void testGetSegmentReplicationStats_WhenNoReplication() { assertEquals(0, replicationStats.maxBytesBehind); } - public void testGetSegmentReplicationStats_WhileOnGoingReplicationAndPrimaryRefreshedToNewCheckPoint() { + public void testGetSegmentReplicationStats_WhileOnGoingReplicationAndPrimaryRefreshedToNewCheckPoint() throws InterruptedException { ShardId shardId = new ShardId("index", "uuid", 0); ReplicationCheckpoint firstReplicationCheckpoint = ReplicationCheckpoint.empty(shardId); - + long baseTime = DateUtils.toLong(Instant.now()); StoreFileMetadata storeFileMetadata1 = new StoreFileMetadata("test-1", 500, "1", Version.LATEST, new BytesRef(500)); StoreFileMetadata storeFileMetadata2 = new StoreFileMetadata("test-2", 500, "1", Version.LATEST, new BytesRef(500)); Map stringStoreFileMetadataMapOne = new HashMap<>(); @@ -232,7 +234,7 @@ public void testGetSegmentReplicationStats_WhileOnGoingReplicationAndPrimaryRefr 1000, "", stringStoreFileMetadataMapOne, - System.nanoTime() - TimeUnit.MINUTES.toNanos(1) + baseTime - 5_000_000 ); IndexShard replicaShard = mock(IndexShard.class); @@ -260,7 +262,7 @@ public void testGetSegmentReplicationStats_WhileOnGoingReplicationAndPrimaryRefr 200, "", stringStoreFileMetadataMapTwo, - System.nanoTime() - TimeUnit.MINUTES.toNanos(1) + baseTime - 1_000_000 ); segmentReplicator.updateReplicationCheckpointStats(thirdReplicationCheckpoint, replicaShard); @@ -276,6 +278,16 @@ public void testGetSegmentReplicationStats_WhileOnGoingReplicationAndPrimaryRefr assertEquals(200, replicationStatsSecond.totalBytesBehind); assertEquals(200, replicationStatsSecond.maxBytesBehind); assertTrue(replicationStatsSecond.maxReplicationLag > 0); + + // shard finished syncing to last checkpoint (sis 3) + when(replicaShard.getLatestReplicationCheckpoint()).thenReturn(thirdReplicationCheckpoint); + segmentReplicator.pruneCheckpointsUpToLastSync(replicaShard); + ReplicationStats finalStats = segmentReplicator.getSegmentReplicationStats(shardId); + assertEquals(0, finalStats.totalBytesBehind); + assertEquals(0, finalStats.maxBytesBehind); + assertEquals(0, finalStats.maxReplicationLag); + // shard is up to date, should not have any tracked stats + assertTrue(segmentReplicator.replicationCheckpointStats.get(shardId).isEmpty()); } public void testGetSegmentReplicationStats_WhenCheckPointReceivedOutOfOrder() { From f9c2fad614ee94d651111f09760eae727411255a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=E1=BA=A7n=20Quang=20D=E1=BB=B1?= Date: Tue, 1 Jul 2025 23:02:33 +0300 Subject: [PATCH 12/51] Replace LegacyESVersion.fromString with Version.fromString (#18567) --------- Signed-off-by: Du Tran --- .../tools/cli/plugin/ListPluginsCommandTests.java | 3 +-- .../opensearch/cluster/node/DiscoveryNodeRole.java | 7 +------ .../src/test/java/org/opensearch/VersionTests.java | 6 +++--- .../cluster/coordination/JoinTaskExecutorTests.java | 3 +-- .../metadata/MetadataIndexUpgradeServiceTests.java | 3 +-- .../opensearch/cluster/node/DiscoveryNodesTests.java | 7 +++---- .../opensearch/common/lucene/uid/VersionsTests.java | 3 +-- .../transport/AbstractSimpleTransportTestCase.java | 12 ++---------- .../yaml/section/ClientYamlTestSectionTests.java | 6 +++--- .../test/rest/yaml/section/SetupSectionTests.java | 6 +++--- .../test/rest/yaml/section/SkipSectionTests.java | 3 +-- 11 files changed, 20 insertions(+), 39 deletions(-) diff --git a/distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/ListPluginsCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/ListPluginsCommandTests.java index 7fcdab907cbf0..36ef7ec253bc2 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/ListPluginsCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/ListPluginsCommandTests.java @@ -33,7 +33,6 @@ package org.opensearch.tools.cli.plugin; import org.apache.lucene.tests.util.LuceneTestCase; -import org.opensearch.LegacyESVersion; import org.opensearch.Version; import org.opensearch.cli.ExitCodes; import org.opensearch.cli.MockTerminal; @@ -271,7 +270,7 @@ public void testExistingIncompatiblePlugin() throws Exception { "version", "1.0", "opensearch.version", - LegacyESVersion.fromString("5.0.0").toString(), + Version.fromString("5.0.0").toString(), "java.version", System.getProperty("java.specification.version"), "classname", diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodeRole.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodeRole.java index 3dc86be816b02..ac4135686486e 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodeRole.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodeRole.java @@ -32,7 +32,6 @@ package org.opensearch.cluster.node; -import org.opensearch.LegacyESVersion; import org.opensearch.Version; import org.opensearch.common.Booleans; import org.opensearch.common.annotation.PublicApi; @@ -349,11 +348,7 @@ public void validateRole(List roles) { * The version that {@link #REMOTE_CLUSTER_CLIENT_ROLE} is introduced. Nodes before this version do not have that role even * they can connect to remote clusters. */ - public static final Version REMOTE_CLUSTER_CLIENT_ROLE_VERSION = LegacyESVersion.fromString("7.8.0"); - - static SortedSet LEGACY_ROLES = Collections.unmodifiableSortedSet( - new TreeSet<>(Arrays.asList(DATA_ROLE, INGEST_ROLE, MASTER_ROLE)) - ); + public static final Version REMOTE_CLUSTER_CLIENT_ROLE_VERSION = Version.fromString("7.8.0"); /** * Represents an unknown role. This can occur if a newer version adds a role that an older version does not know about, or a newer diff --git a/server/src/test/java/org/opensearch/VersionTests.java b/server/src/test/java/org/opensearch/VersionTests.java index 4577dcc6a66f0..61912e7b4891c 100644 --- a/server/src/test/java/org/opensearch/VersionTests.java +++ b/server/src/test/java/org/opensearch/VersionTests.java @@ -96,7 +96,7 @@ public void testVersionComparison() { public void testMin() { assertEquals(VersionUtils.getPreviousVersion(), Version.min(Version.CURRENT, VersionUtils.getPreviousVersion())); - assertEquals(LegacyESVersion.fromString("7.0.1"), Version.min(LegacyESVersion.fromString("7.0.1"), Version.CURRENT)); + assertEquals(Version.fromString("7.0.1"), Version.min(Version.fromString("7.0.1"), Version.CURRENT)); Version version = VersionUtils.randomVersion(random()); Version version1 = VersionUtils.randomVersion(random()); if (version.id <= version1.id) { @@ -189,12 +189,12 @@ public void testIndexCreatedVersion() { } public void testMinCompatVersion() { - Version major = LegacyESVersion.fromString("6.8.0"); + Version major = Version.fromString("6.8.0"); assertThat(Version.fromString("1.0.0").minimumCompatibilityVersion(), equalTo(major)); assertThat(Version.fromString("1.2.0").minimumCompatibilityVersion(), equalTo(major)); assertThat(Version.fromString("1.3.0").minimumCompatibilityVersion(), equalTo(major)); - Version major2x = LegacyESVersion.fromString("7.10.0"); + Version major2x = Version.fromString("7.10.0"); assertThat(Version.fromString("2.0.0").minimumCompatibilityVersion(), equalTo(major2x)); assertThat(Version.fromString("2.2.0").minimumCompatibilityVersion(), equalTo(major2x)); assertThat(Version.fromString("2.3.0").minimumCompatibilityVersion(), equalTo(major2x)); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index ab455a1fbb4e7..f868c6cb9b0df 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -31,7 +31,6 @@ package org.opensearch.cluster.coordination; -import org.opensearch.LegacyESVersion; import org.opensearch.Version; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -137,7 +136,7 @@ public void testPreventJoinClusterWithUnsupportedNodeVersions() { final DiscoveryNode tooLowJoiningNode = new DiscoveryNode( UUIDs.base64UUID(), buildNewFakeTransportAddress(), - LegacyESVersion.fromString("6.7.0") + Version.fromString("6.7.0") ); expectThrows(IllegalStateException.class, () -> { if (randomBoolean()) { diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java index cee95323b8129..6ef1b096960ee 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java @@ -31,7 +31,6 @@ package org.opensearch.cluster.metadata; -import org.opensearch.LegacyESVersion; import org.opensearch.Version; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; @@ -131,7 +130,7 @@ public void testFailUpgrade() { minCompat, Version.max(minCompat, VersionUtils.getPreviousVersion(Version.CURRENT)) ); - Version indexCreated = LegacyESVersion.fromString((minCompat.major - 1) + "." + randomInt(5) + "." + randomInt(5)); + Version indexCreated = Version.fromString((minCompat.major - 1) + "." + randomInt(5) + "." + randomInt(5)); final IndexMetadata metadata = newIndexMeta( "foo", Settings.builder() diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java index 61b86856c9ebc..ceb0c6f1c675b 100644 --- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java +++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java @@ -34,7 +34,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; -import org.opensearch.LegacyESVersion; import org.opensearch.Version; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Setting; @@ -493,7 +492,7 @@ public void testMaxMinNodeVersion() { buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)), - LegacyESVersion.fromString("5.1.0") + Version.fromString("5.1.0") ) ); discoBuilder.add( @@ -503,7 +502,7 @@ public void testMaxMinNodeVersion() { buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)), - LegacyESVersion.fromString("6.3.0") + Version.fromString("6.3.0") ) ); discoBuilder.add( @@ -520,7 +519,7 @@ public void testMaxMinNodeVersion() { discoBuilder.clusterManagerNodeId("name_2"); DiscoveryNodes build = discoBuilder.build(); assertEquals(Version.fromString("1.1.0"), build.getMaxNodeVersion()); - assertEquals(LegacyESVersion.fromString("5.1.0"), build.getMinNodeVersion()); + assertEquals(Version.fromString("5.1.0"), build.getMinNodeVersion()); } private DiscoveryNode buildDiscoveryNodeFromExisting(DiscoveryNode existing, Version newVersion) { diff --git a/server/src/test/java/org/opensearch/common/lucene/uid/VersionsTests.java b/server/src/test/java/org/opensearch/common/lucene/uid/VersionsTests.java index 4a68918587c23..d995a60adfe8d 100644 --- a/server/src/test/java/org/opensearch/common/lucene/uid/VersionsTests.java +++ b/server/src/test/java/org/opensearch/common/lucene/uid/VersionsTests.java @@ -39,7 +39,6 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; -import org.opensearch.LegacyESVersion; import org.opensearch.Version; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; @@ -220,7 +219,7 @@ public void testLuceneVersionOnUnknownVersions() { assertEquals(VersionUtils.getPreviousVersion(Version.fromString("2.1.3")).luceneVersion, version.luceneVersion); // too old version, major should be the oldest supported lucene version minus 1 - version = LegacyESVersion.fromString("5.2.1"); + version = Version.fromString("5.2.1"); assertEquals(VersionUtils.getFirstVersion().luceneVersion.major - 1, version.luceneVersion.major); // future version, should be the same version as today diff --git a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java index 3c351ce0ad2f9..f0f2d452faf8d 100644 --- a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java @@ -40,7 +40,6 @@ import org.apache.lucene.util.CollectionUtil; import org.apache.lucene.util.Constants; import org.opensearch.ExceptionsHelper; -import org.opensearch.LegacyESVersion; import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.ActionListenerResponseHandler; @@ -2199,7 +2198,7 @@ public void testTimeoutPerConnection() throws IOException { public void testHandshakeWithIncompatVersion() { assumeTrue("only tcp transport has a handshake method", serviceA.getOriginalTransport() instanceof TcpTransport); - Version version = LegacyESVersion.fromString("6.0.0"); + Version version = Version.fromString("6.0.0"); try (MockTransportService service = buildService("TS_C", version, Settings.EMPTY)) { service.start(); service.acceptIncomingRequests(); @@ -2225,14 +2224,7 @@ public void testHandshakeUpdatesVersion() throws IOException { service.start(); service.acceptIncomingRequests(); TransportAddress address = service.boundAddress().publishAddress(); - DiscoveryNode node = new DiscoveryNode( - "TS_TPC", - "TS_TPC", - address, - emptyMap(), - emptySet(), - LegacyESVersion.fromString("2.0.0") - ); + DiscoveryNode node = new DiscoveryNode("TS_TPC", "TS_TPC", address, emptyMap(), emptySet(), Version.fromString("2.0.0")); ConnectionProfile.Builder builder = new ConnectionProfile.Builder(); builder.addConnections( 1, diff --git a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSectionTests.java b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSectionTests.java index ee355e307345c..331414a2192fd 100644 --- a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSectionTests.java +++ b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/ClientYamlTestSectionTests.java @@ -32,7 +32,7 @@ package org.opensearch.test.rest.yaml.section; -import org.opensearch.LegacyESVersion; +import org.opensearch.Version; import org.opensearch.common.xcontent.yaml.YamlXContent; import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.XContentParser; @@ -130,8 +130,8 @@ public void testParseTestSectionWithDoSetAndSkipSectionsNoSkip() throws Exceptio assertThat(testSection, notNullValue()); assertThat(testSection.getName(), equalTo("First test section")); assertThat(testSection.getSkipSection(), notNullValue()); - assertThat(testSection.getSkipSection().getLowerVersion(), equalTo(LegacyESVersion.fromString("6.0.0"))); - assertThat(testSection.getSkipSection().getUpperVersion(), equalTo(LegacyESVersion.fromString("6.2.0"))); + assertThat(testSection.getSkipSection().getLowerVersion(), equalTo(Version.fromString("6.0.0"))); + assertThat(testSection.getSkipSection().getUpperVersion(), equalTo(Version.fromString("6.2.0"))); assertThat(testSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259")); assertThat(testSection.getExecutableSections().size(), equalTo(2)); DoSection doSection = (DoSection) testSection.getExecutableSections().get(0); diff --git a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SetupSectionTests.java b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SetupSectionTests.java index 159f5052e7148..7f55df08e4af9 100644 --- a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SetupSectionTests.java +++ b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SetupSectionTests.java @@ -31,7 +31,7 @@ package org.opensearch.test.rest.yaml.section; -import org.opensearch.LegacyESVersion; +import org.opensearch.Version; import org.opensearch.common.xcontent.yaml.YamlXContent; import java.io.IOException; @@ -135,8 +135,8 @@ public void testParseSetupAndSkipSectionNoSkip() throws Exception { assertThat(setupSection, notNullValue()); assertThat(setupSection.getSkipSection().isEmpty(), equalTo(false)); assertThat(setupSection.getSkipSection(), notNullValue()); - assertThat(setupSection.getSkipSection().getLowerVersion(), equalTo(LegacyESVersion.fromString("6.0.0"))); - assertThat(setupSection.getSkipSection().getUpperVersion(), equalTo(LegacyESVersion.fromString("6.3.0"))); + assertThat(setupSection.getSkipSection().getLowerVersion(), equalTo(Version.fromString("6.0.0"))); + assertThat(setupSection.getSkipSection().getUpperVersion(), equalTo(Version.fromString("6.3.0"))); assertThat(setupSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259")); assertThat(setupSection.getExecutableSections().size(), equalTo(2)); assertThat(setupSection.getExecutableSections().get(0), instanceOf(DoSection.class)); diff --git a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SkipSectionTests.java b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SkipSectionTests.java index 9974917d475e8..f3467e8eb532c 100644 --- a/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SkipSectionTests.java +++ b/test/framework/src/test/java/org/opensearch/test/rest/yaml/section/SkipSectionTests.java @@ -32,7 +32,6 @@ package org.opensearch.test.rest.yaml.section; -import org.opensearch.LegacyESVersion; import org.opensearch.Version; import org.opensearch.common.xcontent.yaml.YamlXContent; import org.opensearch.core.common.ParsingException; @@ -72,7 +71,7 @@ public void testSkip() { "foobar" ); assertFalse(section.skip(Version.CURRENT)); - assertTrue(section.skip(LegacyESVersion.fromString("6.0.0"))); + assertTrue(section.skip(Version.fromString("6.0.0"))); section = new SkipSection(randomBoolean() ? null : "6.0.0 - 6.1.0", Collections.singletonList("boom"), "foobar"); assertTrue(section.skip(Version.CURRENT)); } From 16d0fb56cc3bb6dc6f2b41916fb43103a25d75af Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 1 Jul 2025 15:55:48 -0700 Subject: [PATCH 13/51] Inject QueryCollectorSpecFactory from Plugins to create QueryCollectorContext in QueryPhase (#18637) * Initial Commit Signed-off-by: Varun Jain * Javadocs Signed-off-by: Varun Jain * Factory design pattern for QueryCollectorContextSpec Signed-off-by: Varun Jain * Fixing Concurrent Segment Search Signed-off-by: Varun Jain * Remove spec Signed-off-by: Varun Jain * Renaming method Signed-off-by: Varun Jain * Javadoc Signed-off-by: Varun Jain * Removing support method from QueryCollectorContextSpecFactory Signed-off-by: Varun Jain * Adding javadoc Signed-off-by: Varun Jain * Add changelog Signed-off-by: Varun Jain * Add experimentalAPI annotation Signed-off-by: Varun Jain * remove redundant check from getQueryCollectorContextSpec method Signed-off-by: Varun Jain * Add unit tests Signed-off-by: Varun Jain * Remove Test annotation Signed-off-by: Varun Jain * Reformatting Signed-off-by: Varun Jain --------- Signed-off-by: Varun Jain --- CHANGELOG.md | 1 + .../org/opensearch/plugins/SearchPlugin.java | 5 ++ .../org/opensearch/search/SearchModule.java | 6 ++ .../search/query/QueryCollectorArguments.java | 78 ++++++++++++++++++ .../query/QueryCollectorContextSpec.java | 50 ++++++++++++ .../QueryCollectorContextSpecFactory.java | 32 ++++++++ .../QueryCollectorContextSpecRegistry.java | 64 +++++++++++++++ .../opensearch/search/query/QueryPhase.java | 33 +++++++- .../query/QueryCollectorArgumentsTests.java | 49 ++++++++++++ ...ueryCollectorContextSpecRegistryTests.java | 79 +++++++++++++++++++ 10 files changed, 395 insertions(+), 2 deletions(-) create mode 100644 server/src/main/java/org/opensearch/search/query/QueryCollectorArguments.java create mode 100644 server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpec.java create mode 100644 server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecFactory.java create mode 100644 server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecRegistry.java create mode 100644 server/src/test/java/org/opensearch/search/query/QueryCollectorArgumentsTests.java create mode 100644 server/src/test/java/org/opensearch/search/query/QueryCollectorContextSpecRegistryTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 95bca206dd903..5383965916b68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Introduce a new pull-based ingestion plugin for file-based indexing (for local testing) ([#18591](https://github.com/opensearch-project/OpenSearch/pull/18591)) - Add support for search pipeline in search and msearch template ([#18564](https://github.com/opensearch-project/OpenSearch/pull/18564)) - Add BooleanQuery rewrite moving constant-scoring must clauses to filter clauses ([#18510](https://github.com/opensearch-project/OpenSearch/issues/18510)) +- Add functionality for plugins to inject QueryCollectorContext during QueryPhase ([#18637](https://github.com/opensearch-project/OpenSearch/pull/18637)) - Add support for non-timing info in profiler ([#18460](https://github.com/opensearch-project/OpenSearch/issues/18460)) ### Changed diff --git a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java index 710dd32371f83..80a4619f56b64 100644 --- a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java @@ -68,6 +68,7 @@ import org.opensearch.search.deciders.ConcurrentSearchRequestDecider; import org.opensearch.search.fetch.FetchSubPhase; import org.opensearch.search.fetch.subphase.highlight.Highlighter; +import org.opensearch.search.query.QueryCollectorContextSpecFactory; import org.opensearch.search.query.QueryPhaseSearcher; import org.opensearch.search.rescore.Rescorer; import org.opensearch.search.rescore.RescorerBuilder; @@ -227,6 +228,10 @@ default Optional getIndexSearcherExecutorProvider() { return Optional.empty(); } + default List getCollectorContextSpecFactories() { + return emptyList(); + } + /** * Executor service provider */ diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java index 24bdb2d28cba2..cee66b734e76b 100644 --- a/server/src/main/java/org/opensearch/search/SearchModule.java +++ b/server/src/main/java/org/opensearch/search/SearchModule.java @@ -257,6 +257,7 @@ import org.opensearch.search.fetch.subphase.highlight.Highlighter; import org.opensearch.search.fetch.subphase.highlight.PlainHighlighter; import org.opensearch.search.fetch.subphase.highlight.UnifiedHighlighter; +import org.opensearch.search.query.QueryCollectorContextSpecRegistry; import org.opensearch.search.query.QueryPhase; import org.opensearch.search.query.QueryPhaseSearcher; import org.opensearch.search.query.QueryPhaseSearcherWrapper; @@ -350,6 +351,7 @@ public SearchModule(Settings settings, List plugins) { indexSearcherExecutorProvider = registerIndexSearcherExecutorProvider(plugins); namedWriteables.addAll(SortValue.namedWriteables()); concurrentSearchDeciderFactories = registerConcurrentSearchDeciderFactories(plugins); + registerQueryCollectorContextSpec(plugins); } private Collection registerConcurrentSearchDeciderFactories(List plugins) { @@ -1297,6 +1299,10 @@ private SearchPlugin.ExecutorServiceProvider registerIndexSearcherExecutorProvid return provider; } + private void registerQueryCollectorContextSpec(List plugins) { + registerFromPlugin(plugins, SearchPlugin::getCollectorContextSpecFactories, QueryCollectorContextSpecRegistry::registerFactory); + } + public FetchPhase getFetchPhase() { return new FetchPhase(fetchSubPhases); } diff --git a/server/src/main/java/org/opensearch/search/query/QueryCollectorArguments.java b/server/src/main/java/org/opensearch/search/query/QueryCollectorArguments.java new file mode 100644 index 0000000000000..e7bad3db092cd --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/QueryCollectorArguments.java @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Arguments for {@link QueryCollectorContextSpecRegistry} + */ +@ExperimentalApi +public final class QueryCollectorArguments { + private final boolean hasFilterCollector; + + private QueryCollectorArguments(final boolean hasFilterCollector) { + this.hasFilterCollector = hasFilterCollector; + } + + /** + * Whether the query has a filter collector. + * @return true if the query has a filter collector, false otherwise + */ + public boolean hasFilterCollector() { + return hasFilterCollector; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + QueryCollectorArguments queryCollectorArguments = (QueryCollectorArguments) o; + return hasFilterCollector == queryCollectorArguments.hasFilterCollector; + } + + @Override + public int hashCode() { + return Boolean.hashCode(hasFilterCollector); + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + return "QueryCollectorArguments[hasFilterCollector=" + hasFilterCollector + "]"; + } + + /** + * Builder for {@link QueryCollectorArguments} + */ + public static class Builder { + private boolean hasFilterCollector; + + /** + * Set the flag for query has a filter collector. + * @param hasFilterCollector true if the query has a filter collector, false otherwise + * @return Builder instance + */ + public Builder hasFilterCollector(boolean hasFilterCollector) { + this.hasFilterCollector = hasFilterCollector; + return this; + } + + /** + * Build the arguments for the query collector context spec registry. + * @return QueryCollectorArguments instance + */ + public QueryCollectorArguments build() { + return new QueryCollectorArguments(hasFilterCollector); + } + } +} diff --git a/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpec.java b/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpec.java new file mode 100644 index 0000000000000..99379e73ed51f --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpec.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.CollectorManager; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.IOException; + +/** + * interface of QueryCollectorContextSpec + */ +@ExperimentalApi +public interface QueryCollectorContextSpec { + /** + * Context name for QueryCollectorContext + * @return string of context name + */ + String getContextName(); + + /** + * Create collector + * @param in collector + * @return collector + * @throws IOException + */ + Collector create(Collector in) throws IOException; + + /** + * Create collector manager + * @param in collector manager + * @return collector manager + * @throws IOException + */ + CollectorManager createManager(CollectorManager in) throws IOException; + + /** + * Post process query result + * @param result query result + * @throws IOException + */ + void postProcess(QuerySearchResult result) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecFactory.java b/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecFactory.java new file mode 100644 index 0000000000000..b08bebb840343 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecFactory.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Optional; + +/** + * interface of QueryCollectorContext spec factory + */ +@ExperimentalApi +public interface QueryCollectorContextSpecFactory { + /** + * @param searchContext context needed to create collector context spec + * @param queryCollectorArguments arguments to create collector context spec + * @return QueryCollectorContextSpec + * @throws IOException + */ + Optional createQueryCollectorContextSpec( + SearchContext searchContext, + QueryCollectorArguments queryCollectorArguments + ) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecRegistry.java b/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecRegistry.java new file mode 100644 index 0000000000000..413cd63b97856 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecRegistry.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.opensearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * Registry class to load all collector context spec factories during cluster bootstrapping + */ +public final class QueryCollectorContextSpecRegistry { + private static final List registry = new CopyOnWriteArrayList<>(); + + private QueryCollectorContextSpecRegistry() {} + + /** + * Get all collector context spec factories + * @return list of collector context spec factories + */ + public static List getCollectorContextSpecFactories() { + return registry; + } + + /** + * Register factory + * @param factory collector context spec factory defined in plugin + */ + public static void registerFactory(QueryCollectorContextSpecFactory factory) { + registry.add(factory); + } + + /** + * Get collector context spec + * @param searchContext search context + * @param queryCollectorArguments query collector arguments + * @return collector context spec + * @throws IOException + */ + public static Optional getQueryCollectorContextSpec( + final SearchContext searchContext, + final QueryCollectorArguments queryCollectorArguments + ) throws IOException { + Iterator iterator = registry.iterator(); + while (iterator.hasNext()) { + QueryCollectorContextSpecFactory factory = iterator.next(); + Optional spec = factory.createQueryCollectorContextSpec(searchContext, queryCollectorArguments); + if (spec.isEmpty() == false) { + return spec; + } + } + return Optional.empty(); + } +} diff --git a/server/src/main/java/org/opensearch/search/query/QueryPhase.java b/server/src/main/java/org/opensearch/search/query/QueryPhase.java index 58be02cc413dd..ebf8ed0ce3362 100644 --- a/server/src/main/java/org/opensearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/opensearch/search/query/QueryPhase.java @@ -76,6 +76,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.stream.Collectors; @@ -445,9 +446,37 @@ protected boolean searchWithCollector( boolean hasFilterCollector, boolean hasTimeout ) throws IOException { + QueryCollectorContext queryCollectorContext = getQueryCollectorContext(searchContext, hasFilterCollector); + return searchWithCollector(searchContext, searcher, query, collectors, queryCollectorContext, hasFilterCollector, hasTimeout); + } + + private QueryCollectorContext getQueryCollectorContext(SearchContext searchContext, boolean hasFilterCollector) throws IOException { // create the top docs collector last when the other collectors are known - final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext, hasFilterCollector); - return searchWithCollector(searchContext, searcher, query, collectors, topDocsFactory, hasFilterCollector, hasTimeout); + final Optional queryCollectorContextOpt = QueryCollectorContextSpecRegistry.getQueryCollectorContextSpec( + searchContext, + new QueryCollectorArguments.Builder().hasFilterCollector(hasFilterCollector).build() + ).map(queryCollectorContextSpec -> new QueryCollectorContext(queryCollectorContextSpec.getContextName()) { + @Override + Collector create(Collector in) throws IOException { + return queryCollectorContextSpec.create(in); + } + + @Override + CollectorManager createManager(CollectorManager in) + throws IOException { + return queryCollectorContextSpec.createManager(in); + } + + @Override + void postProcess(QuerySearchResult result) throws IOException { + queryCollectorContextSpec.postProcess(result); + } + }); + if (queryCollectorContextOpt.isPresent()) { + return queryCollectorContextOpt.get(); + } else { + return createTopDocsCollectorContext(searchContext, hasFilterCollector); + } } protected boolean searchWithCollector( diff --git a/server/src/test/java/org/opensearch/search/query/QueryCollectorArgumentsTests.java b/server/src/test/java/org/opensearch/search/query/QueryCollectorArgumentsTests.java new file mode 100644 index 0000000000000..a853edf5c8be7 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/query/QueryCollectorArgumentsTests.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.opensearch.test.OpenSearchTestCase; + +public class QueryCollectorArgumentsTests extends OpenSearchTestCase { + + public void testBuilder() { + QueryCollectorArguments args = new QueryCollectorArguments.Builder().hasFilterCollector(true).build(); + + assertTrue(args.hasFilterCollector()); + } + + public void testEquals() { + QueryCollectorArguments args1 = new QueryCollectorArguments.Builder().hasFilterCollector(true).build(); + + QueryCollectorArguments args2 = new QueryCollectorArguments.Builder().hasFilterCollector(true).build(); + + QueryCollectorArguments args3 = new QueryCollectorArguments.Builder().hasFilterCollector(false).build(); + + assertTrue(args1.equals(args2)); // Same values + assertFalse(args1.equals(args3)); // Different values + assertTrue(args1.equals(args1)); // Same object + } + + public void testHashCode() { + QueryCollectorArguments args1 = new QueryCollectorArguments.Builder().hasFilterCollector(true).build(); + + QueryCollectorArguments args2 = new QueryCollectorArguments.Builder().hasFilterCollector(true).build(); + + assertEquals(args1.hashCode(), args2.hashCode()); + assertEquals(args1.hashCode(), args1.hashCode()); // Consistent + } + + public void testToString() { + QueryCollectorArguments args = new QueryCollectorArguments.Builder().hasFilterCollector(true).build(); + + String result = args.toString(); + + assertEquals("QueryCollectorArguments[hasFilterCollector=true]", result); + } +} diff --git a/server/src/test/java/org/opensearch/search/query/QueryCollectorContextSpecRegistryTests.java b/server/src/test/java/org/opensearch/search/query/QueryCollectorContextSpecRegistryTests.java new file mode 100644 index 0000000000000..03fedc3534e82 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/query/QueryCollectorContextSpecRegistryTests.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.opensearch.search.internal.SearchContext; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.Optional; + +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.mockito.Mockito.when; + +public class QueryCollectorContextSpecRegistryTests extends OpenSearchTestCase { + @Mock + private QueryCollectorContextSpecFactory mockFactory1; + + @Mock + private SearchContext mockSearchContext; + + @Mock + private QueryCollectorContextSpec mockSpec; + + @Before + public void setUp() throws Exception { + super.setUp(); + MockitoAnnotations.initMocks(this); + // Clear registry before each test + QueryCollectorContextSpecRegistry.getCollectorContextSpecFactories().clear(); + } + + public void testRegisterFactory() { + QueryCollectorContextSpecRegistry.registerFactory(mockFactory1); + assertEquals(1, QueryCollectorContextSpecRegistry.getCollectorContextSpecFactories().size()); + assertTrue(QueryCollectorContextSpecRegistry.getCollectorContextSpecFactories().contains(mockFactory1)); + } + + public void testGetQueryCollectorContextSpec_WithValidSpec() throws IOException { + + QueryCollectorArguments mockArguments = new QueryCollectorArguments.Builder().build(); + // Given + QueryCollectorContextSpecRegistry.registerFactory(mockFactory1); + when(mockFactory1.createQueryCollectorContextSpec(mockSearchContext, mockArguments)).thenReturn(Optional.of(mockSpec)); + + // When + Optional result = QueryCollectorContextSpecRegistry.getQueryCollectorContextSpec( + mockSearchContext, + mockArguments + ); + + // Then + assertTrue(result.isPresent()); + assertEquals(mockSpec, result.get()); + } + + public void testGetQueryCollectorContextSpec_NoFactories() throws IOException { + + QueryCollectorArguments mockArguments = new QueryCollectorArguments.Builder().build(); + + // When + Optional result = QueryCollectorContextSpecRegistry.getQueryCollectorContextSpec( + mockSearchContext, + mockArguments + ); + + // Then + assertTrue(result.isEmpty()); + } + +} From 36c2d5930ab3c036f19760942f95ea4bdb126137 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 17:37:09 -0700 Subject: [PATCH 14/51] Bump com.azure:azure-storage-blob from 12.30.0 to 12.30.1 in /plugins/repository-azure (#18644) * Bump com.azure:azure-storage-blob in /plugins/repository-azure Bumps [com.azure:azure-storage-blob](https://github.com/Azure/azure-sdk-for-java) from 12.30.0 to 12.30.1. - [Release notes](https://github.com/Azure/azure-sdk-for-java/releases) - [Commits](https://github.com/Azure/azure-sdk-for-java/compare/azure-storage-blob_12.30.0...azure-storage-blob_12.30.1) --- updated-dependencies: - dependency-name: com.azure:azure-storage-blob dependency-version: 12.30.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- .../licenses/azure-storage-blob-12.30.0.jar.sha1 | 1 - .../licenses/azure-storage-blob-12.30.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/azure-storage-blob-12.30.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-storage-blob-12.30.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5383965916b68..81d1fd50defc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.azure:azure-core-http-netty` from 1.15.11 to 1.15.12 ([#18586](https://github.com/opensearch-project/OpenSearch/pull/18586)) - Bump `com.squareup.okio:okio` from 3.13.0 to 3.14.0 ([#18645](https://github.com/opensearch-project/OpenSearch/pull/18645)) - Bump `com.netflix.nebula.ospackage-base` from 11.11.2 to 12.0.0 ([#18646](https://github.com/opensearch-project/OpenSearch/pull/18646)) +- Bump `com.azure:azure-storage-blob` from 12.30.0 to 12.30.1 ([#18644](https://github.com/opensearch-project/OpenSearch/pull/18644)) ### Deprecated diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 5064a737619c1..2946e5a5d8a11 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -56,7 +56,7 @@ dependencies { api "io.netty:netty-resolver-dns:${versions.netty}" api "io.netty:netty-transport-native-unix-common:${versions.netty}" implementation project(':modules:transport-netty4') - api 'com.azure:azure-storage-blob:12.30.0' + api 'com.azure:azure-storage-blob:12.30.1' api 'com.azure:azure-identity:1.14.2' // Start of transitive dependencies for azure-identity api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0' diff --git a/plugins/repository-azure/licenses/azure-storage-blob-12.30.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-blob-12.30.0.jar.sha1 deleted file mode 100644 index 2f6fc7a879e5f..0000000000000 --- a/plugins/repository-azure/licenses/azure-storage-blob-12.30.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a187bbdf04d9d4c0144ef619ba02ce1cd07211ac \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-blob-12.30.1.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-blob-12.30.1.jar.sha1 new file mode 100644 index 0000000000000..34189c82a88ba --- /dev/null +++ b/plugins/repository-azure/licenses/azure-storage-blob-12.30.1.jar.sha1 @@ -0,0 +1 @@ +deaa55c7c985bec01cbbc4fef41d2da3d511dcbc \ No newline at end of file From 46a0045a2301b3bc51e2da9c5e853f69ac6342c2 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Thu, 3 Jul 2025 17:29:44 +0530 Subject: [PATCH 15/51] Fix checkpoint handling to prevent segment replication infinite loop (#18636) * Fix checkpoint handling to prevent segment replication infinite loop Signed-off-by: Ashish Singh * Modify existing test to verify code change behaviour Signed-off-by: Ashish Singh --------- Signed-off-by: Ashish Singh --- .../replication/SegmentReplicationTargetService.java | 6 +++++- .../replication/SegmentReplicationTargetServiceTests.java | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index 1be0224a77b60..d39b5b62781b8 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -49,6 +49,7 @@ import java.io.IOException; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicLong; @@ -347,7 +348,10 @@ public void onReplicationDone(SegmentReplicationState state) { // if we received a checkpoint during the copy event that is ahead of this // try and process it. - processLatestReceivedCheckpoint(replicaShard, thread); + ReplicationCheckpoint latestReceivedCheckpoint = replicator.getPrimaryCheckpoint(replicaShard.shardId()); + if (Objects.nonNull(latestReceivedCheckpoint) && latestReceivedCheckpoint.isAheadOf(receivedCheckpoint)) { + processLatestReceivedCheckpoint(replicaShard, thread); + } } @Override diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index f9c9b451d3c77..f8dc1c3dc971d 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -625,6 +625,7 @@ public void testStartReplicationListenerSuccess() throws InterruptedException { latch.await(2, TimeUnit.SECONDS); verify(spy, (atLeastOnce())).updateVisibleCheckpoint(eq(0L), eq(replicaShard)); + verify(spy, times(1)).processLatestReceivedCheckpoint(any(), any()); } public void testStartReplicationListenerFailure() throws InterruptedException { @@ -851,4 +852,5 @@ public void testProcessCheckpointOnClusterStateUpdate() { spy.clusterChanged(new ClusterChangedEvent("ignored", oldState, newState)); verify(spy, times(1)).processLatestReceivedCheckpoint(eq(replicaShard), any()); } + } From 1e00145b8a46c447384ef5cf6718d7d4fc7e365b Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Thu, 3 Jul 2025 18:42:27 -0700 Subject: [PATCH 16/51] Extend Approximation Framework to other numeric types (#18530) * Extend approximation to other numeric types Signed-off-by: Prudhvi Godithi * Fix basic tests Signed-off-by: Prudhvi Godithi * Fix basic tests Signed-off-by: Prudhvi Godithi * Add Parameterized tests for all numeric types Signed-off-by: Prudhvi Godithi * Fix FieldSortIT Signed-off-by: Prudhvi Godithi * Fix FieldSortIT Signed-off-by: Prudhvi Godithi * Address comments Signed-off-by: Prudhvi Godithi * Update tests and changelog Signed-off-by: Prudhvi Godithi * Update tests and changelog Signed-off-by: Prudhvi Godithi --------- Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 1 + .../opensearch/search/sort/FieldSortIT.java | 17 +- .../index/mapper/NumberFieldMapper.java | 191 ++- .../ApproximatePointRangeQuery.java | 10 + .../index/mapper/NumberFieldTypeTests.java | 24 +- .../index/query/RangeQueryBuilderTests.java | 11 +- .../ApproximatePointRangeQueryTests.java | 1060 ++++++++++------- 7 files changed, 844 insertions(+), 470 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81d1fd50defc2..194fd74c63be4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add BooleanQuery rewrite moving constant-scoring must clauses to filter clauses ([#18510](https://github.com/opensearch-project/OpenSearch/issues/18510)) - Add functionality for plugins to inject QueryCollectorContext during QueryPhase ([#18637](https://github.com/opensearch-project/OpenSearch/pull/18637)) - Add support for non-timing info in profiler ([#18460](https://github.com/opensearch-project/OpenSearch/issues/18460)) +- Extend Approximation Framework to other numeric types ([#18530](https://github.com/opensearch-project/OpenSearch/issues/18530)) ### Changed - Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570)) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java index e8675ff3a6e55..c1a869a43d8aa 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java @@ -934,7 +934,7 @@ public void testSortMissingNumbers() throws Exception { indexRandomForConcurrentSearch("test"); // DOUBLE - logger.info("--> sort with no missing (same as missing _last)"); + logger.info("--> sort with no missing"); SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC)) @@ -944,7 +944,6 @@ public void testSortMissingNumbers() throws Exception { assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L)); assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("3")); - assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("2")); logger.info("--> sort with missing _last"); searchResponse = client().prepareSearch() @@ -983,7 +982,7 @@ public void testSortMissingNumbers() throws Exception { assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); // FLOAT - logger.info("--> sort with no missing (same as missing _last)"); + logger.info("--> sort with no missing"); searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .addSort(SortBuilders.fieldSort("d_value").order(SortOrder.ASC)) @@ -993,7 +992,6 @@ public void testSortMissingNumbers() throws Exception { assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L)); assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("3")); - assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("2")); logger.info("--> sort with missing _last"); searchResponse = client().prepareSearch() @@ -1032,7 +1030,7 @@ public void testSortMissingNumbers() throws Exception { assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); // UNSIGNED_LONG - logger.info("--> sort with no missing (same as missing _last)"); + logger.info("--> sort with no missing"); searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .addSort(SortBuilders.fieldSort("u_value").order(SortOrder.ASC)) @@ -1042,7 +1040,6 @@ public void testSortMissingNumbers() throws Exception { assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L)); assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("3")); - assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("2")); logger.info("--> sort with missing _last"); searchResponse = client().prepareSearch() @@ -1138,7 +1135,7 @@ public void testSortMissingNumbersMinMax() throws Exception { indexRandomForConcurrentSearch("test"); // LONG - logger.info("--> sort with no missing (same as missing _last)"); + logger.info("--> sort with no missing"); SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .addSort(SortBuilders.fieldSort("l_value").order(SortOrder.ASC)) @@ -1177,7 +1174,7 @@ public void testSortMissingNumbersMinMax() throws Exception { assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); // FLOAT - logger.info("--> sort with no missing (same as missing _last)"); + logger.info("--> sort with no missing"); searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .addSort(SortBuilders.fieldSort("d_value").order(SortOrder.ASC)) @@ -1187,7 +1184,6 @@ public void testSortMissingNumbersMinMax() throws Exception { assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L)); assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("3")); - assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("2")); logger.info("--> sort with missing _last"); searchResponse = client().prepareSearch() @@ -1214,7 +1210,7 @@ public void testSortMissingNumbersMinMax() throws Exception { assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); // UNSIGNED_LONG - logger.info("--> sort with no missing (same as missing _last)"); + logger.info("--> sort with no missing"); searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .addSort(SortBuilders.fieldSort("u_value").order(SortOrder.ASC)) @@ -1225,7 +1221,6 @@ public void testSortMissingNumbersMinMax() throws Exception { assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); // The order here could be unstable (depends on document order) since missing == field value assertThat(searchResponse.getHits().getAt(1).getId(), is(oneOf("3", "2"))); - assertThat(searchResponse.getHits().getAt(2).getId(), is(oneOf("2", "3"))); logger.info("--> sort with missing _last"); searchResponse = client().prepareSearch() diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index 61a5308fc3b3c..6512007c9683e 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -102,6 +102,8 @@ public class NumberFieldMapper extends ParametrizedFieldMapper { public static final Setting COERCE_SETTING = Setting.boolSetting("index.mapping.coerce", true, Property.IndexScope); + private static final int APPROX_QUERY_NUMERIC_DIMS = 1; + private static NumberFieldMapper toType(FieldMapper in) { return (NumberFieldMapper) in; } @@ -356,23 +358,50 @@ public Query rangeQuery( } u = HalfFloatPoint.nextDown(u); } - if (isSearchable && hasDocValues) { - Query query = HalfFloatPoint.newRangeQuery(field, l, u); - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery( + + Query dvQuery = hasDocValues + ? SortedNumericDocValuesField.newSlowRangeQuery( field, HalfFloatPoint.halfFloatToSortableShort(l), HalfFloatPoint.halfFloatToSortableShort(u) + ) + : null; + if (isSearchable) { + Query pointRangeQuery = HalfFloatPoint.newRangeQuery(field, l, u); + Query query; + if (dvQuery != null) { + query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery); + if (context.indexSortedOnField(field)) { + query = new IndexSortSortedNumericDocValuesRangeQuery( + field, + HalfFloatPoint.halfFloatToSortableShort(l), + HalfFloatPoint.halfFloatToSortableShort(u), + query + ); + } + } else { + query = pointRangeQuery; + } + return new ApproximateScoreQuery( + query, + new ApproximatePointRangeQuery( + field, + NumberType.HALF_FLOAT.encodePoint(l), + NumberType.HALF_FLOAT.encodePoint(u), + APPROX_QUERY_NUMERIC_DIMS, + ApproximatePointRangeQuery.HALF_FLOAT_FORMAT + ) ); - return new IndexOrDocValuesQuery(query, dvQuery); } - if (hasDocValues) { - return SortedNumericDocValuesField.newSlowRangeQuery( + if (context.indexSortedOnField(field)) { + dvQuery = new IndexSortSortedNumericDocValuesRangeQuery( field, HalfFloatPoint.halfFloatToSortableShort(l), - HalfFloatPoint.halfFloatToSortableShort(u) + HalfFloatPoint.halfFloatToSortableShort(u), + dvQuery ); } - return HalfFloatPoint.newRangeQuery(field, l, u); + return dvQuery; } @Override @@ -503,23 +532,52 @@ public Query rangeQuery( u = FloatPoint.nextDown(u); } } - if (isSearchable && hasDocValues) { - Query query = FloatPoint.newRangeQuery(field, l, u); - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery( + + Query dvQuery = hasDocValues + ? SortedNumericDocValuesField.newSlowRangeQuery( field, NumericUtils.floatToSortableInt(l), NumericUtils.floatToSortableInt(u) + ) + : null; + + if (isSearchable) { + Query pointRangeQuery = FloatPoint.newRangeQuery(field, l, u); + Query query; + if (dvQuery != null) { + query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery); + if (context.indexSortedOnField(field)) { + query = new IndexSortSortedNumericDocValuesRangeQuery( + field, + NumericUtils.floatToSortableInt(l), + NumericUtils.floatToSortableInt(u), + query + ); + } + } else { + query = pointRangeQuery; + } + return new ApproximateScoreQuery( + query, + new ApproximatePointRangeQuery( + field, + FloatPoint.pack(new float[] { l }).bytes, + FloatPoint.pack(new float[] { u }).bytes, + APPROX_QUERY_NUMERIC_DIMS, + ApproximatePointRangeQuery.FLOAT_FORMAT + ) ); - return new IndexOrDocValuesQuery(query, dvQuery); } - if (hasDocValues) { - return SortedNumericDocValuesField.newSlowRangeQuery( + + if (context.indexSortedOnField(field)) { + dvQuery = new IndexSortSortedNumericDocValuesRangeQuery( field, NumericUtils.floatToSortableInt(l), - NumericUtils.floatToSortableInt(u) + NumericUtils.floatToSortableInt(u), + dvQuery ); } - return FloatPoint.newRangeQuery(field, l, u); + return dvQuery; } @Override @@ -628,23 +686,49 @@ public Query rangeQuery( QueryShardContext context ) { return doubleRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> { - if (isSearchable && hasDocValues) { - Query query = DoublePoint.newRangeQuery(field, l, u); - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery( + Query dvQuery = hasDocValues + ? SortedNumericDocValuesField.newSlowRangeQuery( field, NumericUtils.doubleToSortableLong(l), NumericUtils.doubleToSortableLong(u) + ) + : null; + if (isSearchable) { + Query pointRangeQuery = DoublePoint.newRangeQuery(field, l, u); + Query query; + if (dvQuery != null) { + query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery); + if (context.indexSortedOnField(field)) { + query = new IndexSortSortedNumericDocValuesRangeQuery( + field, + NumericUtils.doubleToSortableLong(l), + NumericUtils.doubleToSortableLong(u), + query + ); + } + } else { + query = pointRangeQuery; + } + return new ApproximateScoreQuery( + query, + new ApproximatePointRangeQuery( + field, + DoublePoint.pack(new double[] { l }).bytes, + DoublePoint.pack(new double[] { u }).bytes, + APPROX_QUERY_NUMERIC_DIMS, + ApproximatePointRangeQuery.DOUBLE_FORMAT + ) ); - return new IndexOrDocValuesQuery(query, dvQuery); } - if (hasDocValues) { - return SortedNumericDocValuesField.newSlowRangeQuery( + if (context.indexSortedOnField(field)) { + dvQuery = new IndexSortSortedNumericDocValuesRangeQuery( field, NumericUtils.doubleToSortableLong(l), - NumericUtils.doubleToSortableLong(u) + NumericUtils.doubleToSortableLong(u), + dvQuery ); } - return DoublePoint.newRangeQuery(field, l, u); + return dvQuery; }); } @@ -988,23 +1072,33 @@ public Query rangeQuery( --u; } } - if (isSearchable && hasDocValues) { - Query query = IntPoint.newRangeQuery(field, l, u); - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u); - query = new IndexOrDocValuesQuery(query, dvQuery); - if (context.indexSortedOnField(field)) { - query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query); + Query dvQuery = hasDocValues ? SortedNumericDocValuesField.newSlowRangeQuery(field, l, u) : null; + if (isSearchable) { + Query pointRangeQuery = IntPoint.newRangeQuery(field, l, u); + Query query; + if (dvQuery != null) { + query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery); + if (context.indexSortedOnField(field)) { + query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query); + } + } else { + query = pointRangeQuery; } - return query; + return new ApproximateScoreQuery( + query, + new ApproximatePointRangeQuery( + field, + IntPoint.pack(new int[] { l }).bytes, + IntPoint.pack(new int[] { u }).bytes, + APPROX_QUERY_NUMERIC_DIMS, + ApproximatePointRangeQuery.INT_FORMAT + ) + ); } - if (hasDocValues) { - Query query = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u); - if (context.indexSortedOnField(field)) { - query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query); - } - return query; + if (context.indexSortedOnField(field)) { + dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, dvQuery); } - return IntPoint.newRangeQuery(field, l, u); + return dvQuery; } @Override @@ -1136,11 +1230,10 @@ public Query rangeQuery( field, LongPoint.pack(new long[] { l }).bytes, LongPoint.pack(new long[] { u }).bytes, - new long[] { l }.length, + APPROX_QUERY_NUMERIC_DIMS, ApproximatePointRangeQuery.LONG_FORMAT ) ); - } if (context.indexSortedOnField(field)) { dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, dvQuery); @@ -1257,10 +1350,22 @@ public Query rangeQuery( QueryShardContext context ) { return unsignedLongRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> { - if (isSearchable && hasDocValues) { + if (isSearchable) { Query query = BigIntegerPoint.newRangeQuery(field, l, u); - Query dvQuery = SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery(field, l, u); - return new IndexOrDocValuesQuery(query, dvQuery); + if (hasDocValues) { + Query dvQuery = SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery(field, l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return new ApproximateScoreQuery( + query, + new ApproximatePointRangeQuery( + field, + NumberType.UNSIGNED_LONG.encodePoint(l), + NumberType.UNSIGNED_LONG.encodePoint(u), + APPROX_QUERY_NUMERIC_DIMS, + ApproximatePointRangeQuery.UNSIGNED_LONG_FORMAT + ) + ); } if (hasDocValues) { return SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery(field, l, u); diff --git a/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java b/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java index bd8f637136ea7..e6d11d02eacf6 100644 --- a/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java +++ b/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java @@ -8,10 +8,15 @@ package org.opensearch.search.approximate; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PointValues; +import org.apache.lucene.sandbox.document.BigIntegerPoint; +import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; @@ -40,6 +45,11 @@ */ public class ApproximatePointRangeQuery extends ApproximateQuery { public static final Function LONG_FORMAT = bytes -> Long.toString(LongPoint.decodeDimension(bytes, 0)); + public static final Function INT_FORMAT = bytes -> Integer.toString(IntPoint.decodeDimension(bytes, 0)); + public static final Function HALF_FLOAT_FORMAT = bytes -> Float.toString(HalfFloatPoint.decodeDimension(bytes, 0)); + public static final Function FLOAT_FORMAT = bytes -> Float.toString(FloatPoint.decodeDimension(bytes, 0)); + public static final Function DOUBLE_FORMAT = bytes -> Double.toString(DoublePoint.decodeDimension(bytes, 0)); + public static final Function UNSIGNED_LONG_FORMAT = bytes -> BigIntegerPoint.decodeDimension(bytes, 0).toString(); private int size; diff --git a/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java index 6382f033d74d5..5fc771a422c01 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java @@ -411,10 +411,20 @@ public void testLongRangeQuery() { public void testUnsignedLongRangeQuery() { MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.UNSIGNED_LONG); - Query expected = new IndexOrDocValuesQuery( + Query indexOrDvQuery = new IndexOrDocValuesQuery( BigIntegerPoint.newRangeQuery("field", BigInteger.valueOf(1), BigInteger.valueOf(3)), SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery("field", BigInteger.valueOf(1), BigInteger.valueOf(3)) ); + Query expected = new ApproximateScoreQuery( + indexOrDvQuery, + new ApproximatePointRangeQuery( + "field", + NumberType.UNSIGNED_LONG.encodePoint(BigInteger.valueOf(1)), + NumberType.UNSIGNED_LONG.encodePoint(BigInteger.valueOf(3)), + 1, + ApproximatePointRangeQuery.UNSIGNED_LONG_FORMAT + ) + ); assertEquals(expected, ft.rangeQuery("1", "3", true, true, null, null, null, MOCK_QSC)); MappedFieldType unsearchable = unsearchable(); @@ -443,7 +453,7 @@ public void testUnsignedLongTermsQuery() { public void testDoubleRangeQuery() { MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE); - Query expected = new IndexOrDocValuesQuery( + Query indexOrDvQuery = new IndexOrDocValuesQuery( DoublePoint.newRangeQuery("field", 1d, 3d), SortedNumericDocValuesField.newSlowRangeQuery( "field", @@ -451,6 +461,16 @@ public void testDoubleRangeQuery() { NumericUtils.doubleToSortableLong(3) ) ); + Query expected = new ApproximateScoreQuery( + indexOrDvQuery, + new ApproximatePointRangeQuery( + "field", + DoublePoint.pack(new double[] { 1d }).bytes, + DoublePoint.pack(new double[] { 3d }).bytes, + 1, + ApproximatePointRangeQuery.DOUBLE_FORMAT + ) + ); assertEquals(expected, ft.rangeQuery("1", "3", true, true, null, null, null, MOCK_QSC)); MappedFieldType unsearchable = unsearchable(); diff --git a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java index c67e9f8b3180c..7994852217033 100644 --- a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java @@ -257,8 +257,12 @@ protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query, query ); } else if (expectedFieldName.equals(INT_FIELD_NAME)) { - assertThat(query, instanceOf(IndexOrDocValuesQuery.class)); - query = ((IndexOrDocValuesQuery) query).getIndexQuery(); + assertThat(query, instanceOf(ApproximateScoreQuery.class)); + Query approximationQuery = ((ApproximateScoreQuery) query).getApproximationQuery(); + assertThat(approximationQuery, instanceOf(ApproximateQuery.class)); + Query originalQuery = ((ApproximateScoreQuery) query).getOriginalQuery(); + assertThat(originalQuery, instanceOf(IndexOrDocValuesQuery.class)); + query = ((IndexOrDocValuesQuery) originalQuery).getIndexQuery(); assertThat(query, instanceOf(PointRangeQuery.class)); Integer min = (Integer) queryBuilder.from(); Integer max = (Integer) queryBuilder.to(); @@ -299,6 +303,9 @@ public void testIllegalArguments() { public void testToQueryNumericField() throws IOException { Query parsedQuery = rangeQuery(INT_FIELD_NAME).from(23).to(54).includeLower(true).includeUpper(false).toQuery(createShardContext()); + if (parsedQuery instanceof ApproximateScoreQuery) { + parsedQuery = ((ApproximateScoreQuery) parsedQuery).getOriginalQuery(); + } // since age is automatically registered in data, we encode it as numeric assertThat(parsedQuery, instanceOf(IndexOrDocValuesQuery.class)); parsedQuery = ((IndexOrDocValuesQuery) parsedQuery).getIndexQuery(); diff --git a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java index f02386c5e0d10..1103062379a48 100644 --- a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java +++ b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java @@ -8,13 +8,19 @@ package org.opensearch.search.approximate; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.sandbox.document.BigIntegerPoint; +import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; @@ -29,54 +35,281 @@ import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Collection; +import java.util.function.Function; -import static java.util.Arrays.asList; -import static org.apache.lucene.document.LongPoint.pack; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class ApproximatePointRangeQueryTests extends OpenSearchTestCase { - protected static final String DATE_FIELD_NAME = "mapped_date"; + private final NumericType numericType; + + public ApproximatePointRangeQueryTests(NumericType numericType) { + this.numericType = numericType; + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[][] { + { NumericType.INT }, + { NumericType.LONG }, + { NumericType.HALF_FLOAT }, + { NumericType.FLOAT }, + { NumericType.DOUBLE }, + { NumericType.UNSIGNED_LONG } } + ); + } + + enum NumericType { + INT("int_field", ApproximatePointRangeQuery.INT_FORMAT) { + @Override + byte[] encode(Number value) { + return IntPoint.pack(new int[] { value.intValue() }).bytes; + } + + @Override + void addField(Document doc, String fieldName, Number value) { + doc.add(new IntPoint(fieldName, value.intValue())); + } + + @Override + void addDocValuesField(Document doc, String fieldName, Number value) { + doc.add(new NumericDocValuesField(fieldName, value.intValue())); + } + + @Override + Query rangeQuery(String fieldName, Number lower, Number upper) { + return IntPoint.newRangeQuery(fieldName, lower.intValue(), upper.intValue()); + } + + @Override + SortField.Type getSortFieldType() { + return SortField.Type.INT; + } + }, + LONG("long_field", ApproximatePointRangeQuery.LONG_FORMAT) { + @Override + byte[] encode(Number value) { + return LongPoint.pack(new long[] { value.longValue() }).bytes; + } + + // Add to BKD + @Override + void addField(Document doc, String fieldName, Number value) { + doc.add(new LongPoint(fieldName, value.longValue())); + } + + // Add DocValues + @Override + void addDocValuesField(Document doc, String fieldName, Number value) { + doc.add(new NumericDocValuesField(fieldName, value.longValue())); + } + + @Override + Query rangeQuery(String fieldName, Number lower, Number upper) { + return LongPoint.newRangeQuery(fieldName, lower.longValue(), upper.longValue()); + } + + @Override + SortField.Type getSortFieldType() { + return SortField.Type.LONG; + } + }, + HALF_FLOAT("half_float_field", ApproximatePointRangeQuery.HALF_FLOAT_FORMAT) { + @Override + byte[] encode(Number value) { + byte[] bytes = new byte[HalfFloatPoint.BYTES]; + HalfFloatPoint.encodeDimension(value.floatValue(), bytes, 0); + return bytes; + } + + @Override + void addField(Document doc, String fieldName, Number value) { + doc.add(new HalfFloatPoint(fieldName, value.floatValue())); + } + + @Override + void addDocValuesField(Document doc, String fieldName, Number value) { + doc.add(new NumericDocValuesField(fieldName + "_sort", value.longValue())); + } + + @Override + String getSortFieldName() { + return fieldName + "_sort"; + } + + @Override + Query rangeQuery(String fieldName, Number lower, Number upper) { + return HalfFloatPoint.newRangeQuery(fieldName, lower.floatValue(), upper.floatValue()); + } + + @Override + SortField.Type getSortFieldType() { + return SortField.Type.LONG; + } + }, + FLOAT("float_field", ApproximatePointRangeQuery.FLOAT_FORMAT) { + @Override + byte[] encode(Number value) { + return FloatPoint.pack(new float[] { value.floatValue() }).bytes; + } + + @Override + void addField(Document doc, String fieldName, Number value) { + doc.add(new FloatPoint(fieldName, value.floatValue())); + } + + @Override + void addDocValuesField(Document doc, String fieldName, Number value) { + doc.add(new NumericDocValuesField(fieldName, Float.floatToIntBits(value.floatValue()))); + } + + @Override + Query rangeQuery(String fieldName, Number lower, Number upper) { + return FloatPoint.newRangeQuery(fieldName, lower.floatValue(), upper.floatValue()); + } + + @Override + SortField.Type getSortFieldType() { + return SortField.Type.FLOAT; + } + }, + DOUBLE("double_field", ApproximatePointRangeQuery.DOUBLE_FORMAT) { + @Override + byte[] encode(Number value) { + return DoublePoint.pack(new double[] { value.doubleValue() }).bytes; + } + + @Override + void addField(Document doc, String fieldName, Number value) { + doc.add(new DoublePoint(fieldName, value.doubleValue())); + } + + @Override + void addDocValuesField(Document doc, String fieldName, Number value) { + doc.add(new NumericDocValuesField(fieldName, Double.doubleToLongBits(value.doubleValue()))); + } + + @Override + Query rangeQuery(String fieldName, Number lower, Number upper) { + return DoublePoint.newRangeQuery(fieldName, lower.doubleValue(), upper.doubleValue()); + } + + @Override + SortField.Type getSortFieldType() { + return SortField.Type.DOUBLE; + } + }, + UNSIGNED_LONG("unsigned_long_field", ApproximatePointRangeQuery.UNSIGNED_LONG_FORMAT) { + @Override + byte[] encode(Number value) { + byte[] bytes = new byte[BigIntegerPoint.BYTES]; + BigIntegerPoint.encodeDimension(BigInteger.valueOf(value.longValue()), bytes, 0); + return bytes; + } + + @Override + void addField(Document doc, String fieldName, Number value) { + doc.add(new BigIntegerPoint(fieldName, BigInteger.valueOf(value.longValue()))); + } + + @Override + void addDocValuesField(Document doc, String fieldName, Number value) { + doc.add(new NumericDocValuesField(fieldName + "_sort", value.longValue())); + } + + @Override + Query rangeQuery(String fieldName, Number lower, Number upper) { + return BigIntegerPoint.newRangeQuery( + fieldName, + BigInteger.valueOf(lower.longValue()), + BigInteger.valueOf(upper.longValue()) + ); + } + + @Override + SortField.Type getSortFieldType() { + return SortField.Type.LONG; + } + + @Override + String getSortFieldName() { + return fieldName + "_sort"; + } + }; + + final String fieldName; + final Function format; + + NumericType(String fieldName, Function format) { + this.fieldName = fieldName; + this.format = format; + } + + abstract byte[] encode(Number value); + + abstract void addField(Document doc, String fieldName, Number value); + + abstract void addDocValuesField(Document doc, String fieldName, Number value); + + abstract Query rangeQuery(String fieldName, Number lower, Number upper); + + abstract SortField.Type getSortFieldType(); + + String getSortFieldName() { + return fieldName; + } + + long getMinTestValue() { + return this == UNSIGNED_LONG ? 0 : -100; + } + } public void testApproximateRangeEqualsActualRange() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { int dims = 1; - - long[] scratch = new long[dims]; - for (int i = 0; i < 100; i++) { - int numPoints = RandomNumbers.randomIntBetween(random(), 1, 10); + int numDocs = RandomNumbers.randomIntBetween(random(), 1500, 3000); + for (int i = 0; i < numDocs; i++) { + int numValues = RandomNumbers.randomIntBetween(random(), 1, 10); Document doc = new Document(); - for (int j = 0; j < numPoints; j++) { - for (int v = 0; v < dims; v++) { - scratch[v] = RandomNumbers.randomLongBetween(random(), 0, 100); - } - doc.add(new LongPoint("point", scratch)); + for (int j = 0; j < numValues; j++) { + long randomValue = RandomNumbers.randomLongBetween(random(), 0, 200); + numericType.addField(doc, numericType.fieldName, randomValue); } iw.addDocument(doc); + if (random().nextInt(20) == 0) { + iw.flush(); + } } iw.flush(); + if (random().nextBoolean()) { + iw.forceMerge(1); + } try (IndexReader reader = iw.getReader()) { - try { - long lower = RandomNumbers.randomLongBetween(random(), -100, 200); - long upper = lower + RandomNumbers.randomLongBetween(random(), 0, 100); - Query approximateQuery = new ApproximatePointRangeQuery( - "point", - pack(lower).bytes, - pack(upper).bytes, - dims, - ApproximatePointRangeQuery.LONG_FORMAT - ); - Query query = LongPoint.newRangeQuery("point", lower, upper); - IndexSearcher searcher = new IndexSearcher(reader); - TopDocs topDocs = searcher.search(approximateQuery, 10); - TopDocs topDocs1 = searcher.search(query, 10); - assertEquals(topDocs.totalHits, topDocs1.totalHits); - } catch (IOException e) { - throw new RuntimeException(e); - } - + long lower = RandomNumbers.randomLongBetween(random(), numericType.getMinTestValue(), 200); + long upper = RandomNumbers.randomLongBetween(random(), lower, lower + 150); + int searchSize = RandomNumbers.randomIntBetween(random(), 10, 50); + Query approximateQuery = new ApproximatePointRangeQuery( + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), + dims, + numericType.format + ); + Query exactQuery = numericType.rangeQuery(numericType.fieldName, lower, upper); + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs topDocs = searcher.search(approximateQuery, searchSize); + TopDocs topDocs1 = searcher.search(exactQuery, searchSize); + assertEquals( + "Approximate and exact queries should return same number of docs", + topDocs1.scoreDocs.length, + topDocs.scoreDocs.length + ); } } } @@ -86,283 +319,297 @@ public void testApproximateRangeWithDefaultSize() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { int dims = 1; - - long[] scratch = new long[dims]; int numPoints = 1000; for (int i = 0; i < numPoints; i++) { Document doc = new Document(); - for (int v = 0; v < dims; v++) { - scratch[v] = i; - } - doc.add(new LongPoint("point", scratch)); + numericType.addField(doc, numericType.fieldName, i); iw.addDocument(doc); if (i % 15 == 0) iw.flush(); } iw.flush(); try (IndexReader reader = iw.getReader()) { - try { - long lower = 0; - long upper = 1000; - Query approximateQuery = new ApproximatePointRangeQuery( - "point", - pack(lower).bytes, - pack(upper).bytes, - dims, - ApproximatePointRangeQuery.LONG_FORMAT - ); - IndexSearcher searcher = new IndexSearcher(reader); - TopDocs topDocs = searcher.search(approximateQuery, 10); - assertEquals(topDocs.totalHits, new TotalHits(1000, TotalHits.Relation.EQUAL_TO)); - } catch (IOException e) { - throw new RuntimeException(e); - } - + long lower = 0; + long upper = 1000; + Query approximateQuery = new ApproximatePointRangeQuery( + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), + dims, + numericType.format + ); + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs topDocs = searcher.search(approximateQuery, 10); + assertEquals(topDocs.totalHits, new TotalHits(1000, TotalHits.Relation.EQUAL_TO)); } } } } - public void testApproximateRangeWithSizeDefault() throws IOException { + public void testApproximateRangeShortCircuitAscSort() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { int dims = 1; - - long[] scratch = new long[dims]; - int numPoints = 1000; + int numPoints = RandomNumbers.randomIntBetween(random(), 1000, 3000); for (int i = 0; i < numPoints; i++) { Document doc = new Document(); - for (int v = 0; v < dims; v++) { - scratch[v] = i; - } - doc.add(new LongPoint("point", scratch)); + numericType.addField(doc, numericType.fieldName, i); + numericType.addDocValuesField(doc, numericType.fieldName, i); iw.addDocument(doc); + // Randomly flush + if (random().nextInt(20) == 0) { + iw.flush(); + } } iw.flush(); - iw.forceMerge(1); - final int size = 10; + if (random().nextBoolean()) { + iw.forceMerge(1); + } try (IndexReader reader = iw.getReader()) { - try { - long lower = 0; - long upper = 45; - Query approximateQuery = new ApproximatePointRangeQuery( - "point", - pack(lower).bytes, - pack(upper).bytes, - dims, - size, - null, - ApproximatePointRangeQuery.LONG_FORMAT - ); - IndexSearcher searcher = new IndexSearcher(reader); - TopDocs topDocs = searcher.search(approximateQuery, size); - assertEquals(size, topDocs.scoreDocs.length); - } catch (IOException e) { - throw new RuntimeException(e); + long lower = RandomNumbers.randomLongBetween(random(), 0, 50); + long upper = RandomNumbers.randomLongBetween(random(), lower + 10, Math.min(lower + 100, numPoints - 1)); + final int size = RandomNumbers.randomIntBetween(random(), 5, 20); + Query approximateQuery = new ApproximatePointRangeQuery( + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), + dims, + size, + SortOrder.ASC, + numericType.format + ); + Query exactQuery = numericType.rangeQuery(numericType.fieldName, lower, upper); + IndexSearcher searcher = new IndexSearcher(reader); + Sort sort = new Sort(new SortField(numericType.getSortFieldName(), numericType.getSortFieldType())); + TopDocs topDocs = searcher.search(approximateQuery, size, sort); + TopDocs topDocs1 = searcher.search(exactQuery, size, sort); + int compareSize = Math.min(size, Math.min(topDocs.scoreDocs.length, topDocs1.scoreDocs.length)); + for (int i = 0; i < compareSize; i++) { + assertEquals("Mismatch at doc index " + i, topDocs.scoreDocs[i].doc, topDocs1.scoreDocs[i].doc); } - } } } } - public void testApproximateRangeWithSizeOverDefault() throws IOException { + public void testApproximateRangeShortCircuitDescSort() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { int dims = 1; - - long[] scratch = new long[dims]; - int numPoints = 15000; + int numPoints = RandomNumbers.randomIntBetween(random(), 1000, 3000); for (int i = 0; i < numPoints; i++) { Document doc = new Document(); - for (int v = 0; v < dims; v++) { - scratch[v] = i; - } - doc.add(new LongPoint("point", scratch)); + numericType.addField(doc, numericType.fieldName, i); + numericType.addDocValuesField(doc, numericType.fieldName, i); iw.addDocument(doc); + if (random().nextInt(20) == 0) { + iw.flush(); + } } iw.flush(); - iw.forceMerge(1); + if (random().nextBoolean()) { + iw.forceMerge(1); + } try (IndexReader reader = iw.getReader()) { - try { - long lower = 0; - long upper = 12000; - long maxHits = 12001; - final int size = 11000; - Query approximateQuery = new ApproximatePointRangeQuery( - "point", - pack(lower).bytes, - pack(upper).bytes, - dims, - size, - null, - ApproximatePointRangeQuery.LONG_FORMAT - ); - IndexSearcher searcher = new IndexSearcher(reader); - TopDocs topDocs = searcher.search(approximateQuery, size); - - if (topDocs.totalHits.relation() == Relation.EQUAL_TO) { - assertEquals(topDocs.totalHits.value(), size); - } else { - assertTrue(11000 <= topDocs.totalHits.value()); - assertTrue(maxHits >= topDocs.totalHits.value()); - } - } catch (IOException e) { - throw new RuntimeException(e); + long lower = RandomNumbers.randomLongBetween(random(), numPoints - 100, numPoints - 20); + long upper = RandomNumbers.randomLongBetween(random(), lower + 10, numPoints - 1); + final int size = RandomNumbers.randomIntBetween(random(), 5, 20); + Query approximateQuery = new ApproximatePointRangeQuery( + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), + dims, + size, + SortOrder.DESC, + numericType.format + ); + Query exactQuery = numericType.rangeQuery(numericType.fieldName, lower, upper); + IndexSearcher searcher = new IndexSearcher(reader); + Sort sort = new Sort(new SortField(numericType.getSortFieldName(), numericType.getSortFieldType(), true)); + TopDocs topDocs = searcher.search(approximateQuery, size, sort); + TopDocs topDocs1 = searcher.search(exactQuery, size, sort); + int compareSize = Math.min(size, Math.min(topDocs.scoreDocs.length, topDocs1.scoreDocs.length)); + for (int i = 0; i < compareSize; i++) { + assertEquals("Mismatch at doc index " + i, topDocs.scoreDocs[i].doc, topDocs1.scoreDocs[i].doc); } - } } } } - public void testApproximateRangeShortCircuit() throws IOException { + public void testApproximateRangeWithSizeDefault() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { int dims = 1; - - long[] scratch = new long[dims]; - int numPoints = 1000; + int numPoints = RandomNumbers.randomIntBetween(random(), 1000, 3000); for (int i = 0; i < numPoints; i++) { Document doc = new Document(); - for (int v = 0; v < dims; v++) { - scratch[v] = i; - } - doc.add(new LongPoint("point", scratch)); + numericType.addField(doc, numericType.fieldName, i); iw.addDocument(doc); - if (i % 10 == 0) iw.flush(); + if (random().nextInt(20) == 0) { + iw.flush(); + } } iw.flush(); - iw.forceMerge(1); + if (random().nextBoolean()) { + iw.forceMerge(1); + } + final int size = RandomNumbers.randomIntBetween(random(), 5, 20); try (IndexReader reader = iw.getReader()) { - try { - long lower = 0; - long upper = 100; - final int size = 10; - Query approximateQuery = new ApproximatePointRangeQuery( - "point", - pack(lower).bytes, - pack(upper).bytes, - dims, - size, - null, - ApproximatePointRangeQuery.LONG_FORMAT - ); - Query query = LongPoint.newRangeQuery("point", lower, upper); - - IndexSearcher searcher = new IndexSearcher(reader); - TopDocs topDocs = searcher.search(approximateQuery, size); - TopDocs topDocs1 = searcher.search(query, size); - assertEquals(size, topDocs.scoreDocs.length); - assertEquals(size, topDocs1.scoreDocs.length); - assertEquals(topDocs1.totalHits.value(), 101); - } catch (IOException e) { - throw new RuntimeException(e); - } - + long lower = RandomNumbers.randomLongBetween(random(), 0, numPoints / 2); + long upper = RandomNumbers.randomLongBetween(random(), lower + size * 2, numPoints - 1); + Query approximateQuery = new ApproximatePointRangeQuery( + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), + dims, + size, + null, + numericType.format + ); + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs topDocs = searcher.search(approximateQuery, size); + assertEquals(size, topDocs.scoreDocs.length); } } } } - public void testApproximateRangeShortCircuitAscSort() throws IOException { + public void testApproximateRangeWithSizeOverDefault() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { int dims = 1; - - long[] scratch = new long[dims]; - int numPoints = 1000; + int numPoints = RandomNumbers.randomIntBetween(random(), 15000, 20000); for (int i = 0; i < numPoints; i++) { Document doc = new Document(); - for (int v = 0; v < dims; v++) { - scratch[v] = i; + numericType.addField(doc, numericType.fieldName, i); + iw.addDocument(doc); + if (random().nextInt(100) == 0) { + iw.flush(); } - iw.addDocument(asList(new LongPoint("point", scratch[0]), new NumericDocValuesField("point", scratch[0]))); } iw.flush(); - iw.forceMerge(1); + if (random().nextBoolean()) { + iw.forceMerge(1); + } try (IndexReader reader = iw.getReader()) { - try { - long lower = 0; - long upper = 20; - final int size = 10; - Query approximateQuery = new ApproximatePointRangeQuery( - "point", - pack(lower).bytes, - pack(upper).bytes, - dims, - size, - SortOrder.ASC, - ApproximatePointRangeQuery.LONG_FORMAT - ); - Query query = LongPoint.newRangeQuery("point", lower, upper); - - IndexSearcher searcher = new IndexSearcher(reader); - Sort sort = new Sort(new SortField("point", SortField.Type.LONG)); - TopDocs topDocs = searcher.search(approximateQuery, size, sort); - TopDocs topDocs1 = searcher.search(query, size, sort); - - assertEquals(topDocs.scoreDocs[0].doc, topDocs1.scoreDocs[0].doc); - assertEquals(topDocs.scoreDocs[1].doc, topDocs1.scoreDocs[1].doc); - assertEquals(topDocs.scoreDocs[2].doc, topDocs1.scoreDocs[2].doc); - assertEquals(topDocs.scoreDocs[3].doc, topDocs1.scoreDocs[3].doc); - assertEquals(topDocs.scoreDocs[4].doc, topDocs1.scoreDocs[4].doc); - assertEquals(topDocs.scoreDocs[5].doc, topDocs1.scoreDocs[5].doc); - - } catch (IOException e) { - throw new RuntimeException(e); + final int size = RandomNumbers.randomIntBetween(random(), 11000, 13000); + long lower = RandomNumbers.randomLongBetween(random(), 0, 1000); + long upper = RandomNumbers.randomLongBetween(random(), lower + size, Math.min(lower + size + 2000, numPoints - 1)); + Query approximateQuery = new ApproximatePointRangeQuery( + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), + dims, + size, + null, + numericType.format + ); + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs topDocs = searcher.search(approximateQuery, size); + if (topDocs.totalHits.relation() == Relation.EQUAL_TO) { + assertEquals(topDocs.totalHits.value(), size); + } else { + assertTrue(size <= topDocs.totalHits.value()); } + } + } + } + } + public void testApproximateRangeShortCircuit() throws IOException { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { + int dims = 1; + int numPoints = RandomNumbers.randomIntBetween(random(), 3000, 10000); + for (int i = 0; i < numPoints; i++) { + Document doc = new Document(); + numericType.addField(doc, numericType.fieldName, i); + iw.addDocument(doc); + if (random().nextInt(20) == 0) { + iw.flush(); + } + } + iw.flush(); + if (random().nextBoolean()) { + iw.forceMerge(1); + } + try (IndexReader reader = iw.getReader()) { + final int size = RandomNumbers.randomIntBetween(random(), 5, 20); + long lower = RandomNumbers.randomLongBetween(random(), 0, numPoints / 2); + long upper = RandomNumbers.randomLongBetween(random(), lower + size * 2, Math.min(lower + 200, numPoints - 1)); + Query approximateQuery = new ApproximatePointRangeQuery( + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), + dims, + size, + null, + numericType.format + ); + Query exactQuery = numericType.rangeQuery(numericType.fieldName, lower, upper); + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs exactAllDocs = searcher.search(exactQuery, numPoints); + long actualHitsInRange = exactAllDocs.totalHits.value(); + TopDocs topDocs = searcher.search(approximateQuery, size); + TopDocs topDocs1 = searcher.search(exactQuery, size); + assertEquals(size, topDocs.scoreDocs.length); + assertEquals(size, topDocs1.scoreDocs.length); + assertEquals(actualHitsInRange, topDocs1.totalHits.value()); + assertTrue("Approximate query should find at least 'size' documents", topDocs.totalHits.value() >= size); } } } } public void testSize() { + long lower = RandomNumbers.randomLongBetween(random(), 0, 100); + long upper = RandomNumbers.randomLongBetween(random(), lower + 10, lower + 1000); ApproximatePointRangeQuery query = new ApproximatePointRangeQuery( - "point", - pack(0).bytes, - pack(20).bytes, + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), 1, - ApproximatePointRangeQuery.LONG_FORMAT + numericType.format ); assertEquals(query.getSize(), 10_000); - - query.setSize(100); - assertEquals(query.getSize(), 100); - + int newSize = RandomNumbers.randomIntBetween(random(), 50, 500); + query.setSize(newSize); + assertEquals(query.getSize(), newSize); } public void testSortOrder() { + long lower = RandomNumbers.randomLongBetween(random(), 0, 100); + long upper = RandomNumbers.randomLongBetween(random(), lower + 10, lower + 1000); ApproximatePointRangeQuery query = new ApproximatePointRangeQuery( - "point", - pack(0).bytes, - pack(20).bytes, + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), 1, - ApproximatePointRangeQuery.LONG_FORMAT + numericType.format ); assertNull(query.getSortOrder()); - - query.setSortOrder(SortOrder.ASC); - assertEquals(query.getSortOrder(), SortOrder.ASC); + SortOrder sortOrder = random().nextBoolean() ? SortOrder.ASC : SortOrder.DESC; + query.setSortOrder(sortOrder); + assertEquals(query.getSortOrder(), sortOrder); } public void testCanApproximate() { + long lower = RandomNumbers.randomLongBetween(random(), 0, 100); + long upper = RandomNumbers.randomLongBetween(random(), lower + 10, lower + 1000); ApproximatePointRangeQuery query = new ApproximatePointRangeQuery( - "point", - pack(0).bytes, - pack(20).bytes, + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), 1, - ApproximatePointRangeQuery.LONG_FORMAT + numericType.format ); - assertFalse(query.canApproximate(null)); - ApproximatePointRangeQuery queryCanApproximate = new ApproximatePointRangeQuery( - "point", - pack(0).bytes, - pack(20).bytes, + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), 1, - ApproximatePointRangeQuery.LONG_FORMAT + numericType.format ) { public boolean canApproximate(SearchContext context) { return true; @@ -373,70 +620,29 @@ public boolean canApproximate(SearchContext context) { } public void testCannotApproximateWithTrackTotalHits() { + long lower = RandomNumbers.randomLongBetween(random(), 0, 100); + long upper = RandomNumbers.randomLongBetween(random(), lower + 10, lower + 1000); ApproximatePointRangeQuery query = new ApproximatePointRangeQuery( - "point", - pack(0).bytes, - pack(20).bytes, + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), 1, - ApproximatePointRangeQuery.LONG_FORMAT + numericType.format ); - SearchContext mockContext = mock(SearchContext.class); + SearchContext mockContext = mock(org.opensearch.search.internal.SearchContext.class); when(mockContext.trackTotalHitsUpTo()).thenReturn(SearchContext.TRACK_TOTAL_HITS_ACCURATE); assertFalse(query.canApproximate(mockContext)); - when(mockContext.trackTotalHitsUpTo()).thenReturn(SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO); + int trackTotalHitsUpTo = RandomNumbers.randomIntBetween(random(), 1000, 20000); + when(mockContext.trackTotalHitsUpTo()).thenReturn(trackTotalHitsUpTo); when(mockContext.aggregations()).thenReturn(null); - when(mockContext.from()).thenReturn(0); - when(mockContext.size()).thenReturn(10); + int from = RandomNumbers.randomIntBetween(random(), 0, 100); + int size = RandomNumbers.randomIntBetween(random(), 10, 100); + when(mockContext.from()).thenReturn(from); + when(mockContext.size()).thenReturn(size); when(mockContext.request()).thenReturn(null); assertTrue(query.canApproximate(mockContext)); - } - - public void testApproximateRangeShortCircuitDescSort() throws IOException { - try (Directory directory = newDirectory()) { - try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { - int dims = 1; - long[] scratch = new long[dims]; - int numPoints = 1000; - for (int i = 0; i < numPoints; i++) { - for (int v = 0; v < dims; v++) { - scratch[v] = i; - } - iw.addDocument(asList(new LongPoint("point", scratch[0]), new NumericDocValuesField("point", scratch[0]))); - } - iw.flush(); - iw.forceMerge(1); - try (IndexReader reader = iw.getReader()) { - try { - long lower = 980; - long upper = 999; - final int size = 10; - Query approximateQuery = new ApproximatePointRangeQuery( - "point", - pack(lower).bytes, - pack(upper).bytes, - dims, - size, - SortOrder.DESC, - ApproximatePointRangeQuery.LONG_FORMAT - ); - Query query = LongPoint.newRangeQuery("point", lower, upper); - - IndexSearcher searcher = new IndexSearcher(reader); - Sort sort = new Sort(new SortField("point", SortField.Type.LONG, true)); // true for DESC - TopDocs topDocs = searcher.search(approximateQuery, size, sort); - TopDocs topDocs1 = searcher.search(query, size, sort); - - // Verify we got the highest values first (DESC order) - for (int i = 0; i < size; i++) { - assertEquals("Mismatch at doc index " + i, topDocs.scoreDocs[i].doc, topDocs1.scoreDocs[i].doc); - } - - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - } + int expectedSize = Math.max(from + size, trackTotalHitsUpTo) + 1; + assertEquals(expectedSize, query.getSize()); } // Test to cover the left child traversal in intersectRight with CELL_INSIDE_QUERY @@ -444,40 +650,43 @@ public void testIntersectRightLeftChildTraversal() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { int dims = 1; - long[] scratch = new long[dims]; - // Create a dataset that will create a multi-level BKD tree - // We need enough documents to create internal nodes (not just leaves) - int numPoints = 2000; + int numPoints = RandomNumbers.randomIntBetween(random(), 2000, 5000); for (int i = 0; i < numPoints; i++) { Document doc = new Document(); - scratch[0] = i; - doc.add(new LongPoint("point", scratch[0])); - iw.addDocument(asList(new LongPoint("point", scratch[0]), new NumericDocValuesField("point", scratch[0]))); - if (i % 100 == 0) { + numericType.addField(doc, numericType.fieldName, i); + numericType.addDocValuesField(doc, numericType.fieldName, i); + iw.addDocument(doc); + if (i % RandomNumbers.randomIntBetween(random(), 50, 200) == 0) { iw.flush(); } } iw.flush(); - iw.forceMerge(1); - + if (random().nextBoolean()) { + iw.forceMerge(1); + } try (IndexReader reader = iw.getReader()) { - // Query that will match many documents and require tree traversal - long lower = 1000; - long upper = 1999; - final int size = 50; + // To test upper half of the data to test the DESC + long lower = RandomNumbers.randomLongBetween(random(), numPoints / 2, numPoints - 200); + long upper = RandomNumbers.randomLongBetween(random(), lower + 100, numPoints - 1); + final int size = RandomNumbers.randomIntBetween(random(), 30, 100); ApproximatePointRangeQuery query = new ApproximatePointRangeQuery( - "point", - pack(lower).bytes, - pack(upper).bytes, + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), dims, - 50, // Small size to ensure we hit the left child traversal condition + size, SortOrder.DESC, - ApproximatePointRangeQuery.LONG_FORMAT + numericType.format ); + IndexSearcher searcher = new IndexSearcher(reader); - Sort sort = new Sort(new SortField("point", SortField.Type.LONG, true)); // DESC + Sort sort = new Sort(new SortField(numericType.getSortFieldName(), numericType.getSortFieldType(), true)); TopDocs topDocs = searcher.search(query, size, sort); - assertEquals("Should return exactly size value documents", size, topDocs.scoreDocs.length); + assertEquals( + "Should return exactly min(size, hits) documents", + (int) Math.min(size, (upper - lower + 1)), + topDocs.scoreDocs.length + ); } } } @@ -488,38 +697,39 @@ public void testIntersectRightCellInsideQueryLeaf() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { int dims = 1; - long[] scratch = new long[dims]; - // Create a smaller dataset that will result in leaf nodes that are completely inside the query range - for (int i = 900; i <= 999; i++) { - scratch[0] = i; - iw.addDocument(asList(new LongPoint("point", scratch[0]), new NumericDocValuesField("point", scratch[0]))); + int dataStart = RandomNumbers.randomIntBetween(random(), 5000, 15000); + int dataEnd = dataStart + RandomNumbers.randomIntBetween(random(), 1000, 2000); + int numDocs = dataEnd - dataStart + 1; + for (int i = dataStart; i <= dataEnd; i++) { + Document doc = new Document(); + numericType.addField(doc, numericType.fieldName, i); + numericType.addDocValuesField(doc, numericType.fieldName, i); + iw.addDocument(doc); } iw.flush(); - iw.forceMerge(1); - + if (random().nextBoolean()) { + iw.forceMerge(1); + } try (IndexReader reader = iw.getReader()) { - // Query that completely contains all documents (CELL_INSIDE_QUERY) - long lower = 800; - long upper = 1100; - final int size = 200; - final int returnSize = 100; + // Create a query range that fully encompasses the data + // This ensures CELL_INSIDE_QUERY condition + long lower = dataStart - RandomNumbers.randomIntBetween(random(), 50, 200); + long upper = dataEnd + RandomNumbers.randomIntBetween(random(), 50, 200); + final int size = RandomNumbers.randomIntBetween(random(), numDocs + 50, numDocs + 200); ApproximatePointRangeQuery query = new ApproximatePointRangeQuery( - "point", - pack(lower).bytes, - pack(upper).bytes, + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), dims, - 200, + size, SortOrder.DESC, - ApproximatePointRangeQuery.LONG_FORMAT + numericType.format ); - IndexSearcher searcher = new IndexSearcher(reader); - Sort sort = new Sort(new SortField("point", SortField.Type.LONG, true)); // DESC + Sort sort = new Sort(new SortField(numericType.getSortFieldName(), numericType.getSortFieldType(), true)); TopDocs topDocs = searcher.search(query, size, sort); - - assertEquals("Should find all documents", returnSize, topDocs.totalHits.value()); - // Should return all the indexed point values from 900 to 999 which tests CELL_INSIDE_QUERY - assertEquals("Should return exactly return size value documents", returnSize, topDocs.scoreDocs.length); + assertEquals("Should find all documents", numDocs, topDocs.totalHits.value()); + assertEquals("Should return exactly all documents", numDocs, topDocs.scoreDocs.length); } } } @@ -530,84 +740,90 @@ public void testIntersectRightCellOutsideQuery() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { int dims = 1; - long[] scratch = new long[dims]; - // Create documents in two separate ranges to ensure some cells are outside query - // Range 1: 0-99 - for (int i = 0; i < 100; i++) { - scratch[0] = i; - iw.addDocument(asList(new LongPoint("point", scratch[0]), new NumericDocValuesField("point", scratch[0]))); + int range1Start = RandomNumbers.randomIntBetween(random(), 0, 1000); + int range1End = range1Start + RandomNumbers.randomIntBetween(random(), 1000, 15000); + for (int i = range1Start; i <= range1End; i++) { + Document doc = new Document(); + numericType.addField(doc, numericType.fieldName, i); + numericType.addDocValuesField(doc, numericType.fieldName, i); + iw.addDocument(doc); } - // Range 2: 500-599 (gap ensures some tree nodes will be completely outside query) - for (int i = 500; i < 600; i++) { - scratch[0] = i; - iw.addDocument(asList(new LongPoint("point", scratch[0]), new NumericDocValuesField("point", scratch[0]))); + int gapSize = RandomNumbers.randomIntBetween(random(), 200, 400); + int range2Start = range1End + gapSize; + int range2End = range2Start + RandomNumbers.randomIntBetween(random(), 50, 150); + for (int i = range2Start; i <= range2End; i++) { + Document doc = new Document(); + numericType.addField(doc, numericType.fieldName, i); + numericType.addDocValuesField(doc, numericType.fieldName, i); + iw.addDocument(doc); } iw.flush(); - iw.forceMerge(1); - + if (random().nextBoolean()) { + iw.forceMerge(1); + } try (IndexReader reader = iw.getReader()) { - // Query only the middle range - this should create CELL_OUTSIDE_QUERY for some nodes - long lower = 200; - long upper = 400; - final int size = 50; + long lower = range1End + RandomNumbers.randomIntBetween(random(), 10, gapSize / 2); + long upper = range2Start - RandomNumbers.randomIntBetween(random(), 10, gapSize / 2); + final int size = RandomNumbers.randomIntBetween(random(), 20, 100); + ApproximatePointRangeQuery query = new ApproximatePointRangeQuery( - "point", - pack(lower).bytes, - pack(upper).bytes, + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), dims, size, SortOrder.DESC, - ApproximatePointRangeQuery.LONG_FORMAT + numericType.format ); - IndexSearcher searcher = new IndexSearcher(reader); - Sort sort = new Sort(new SortField("point", SortField.Type.LONG, true)); // DESC + Sort sort = new Sort(new SortField(numericType.getSortFieldName(), numericType.getSortFieldType(), true)); TopDocs topDocs = searcher.search(query, size, sort); - - // Should find no documents since our query range (200-400) has no documents assertEquals("Should find no documents in the gap range", 0, topDocs.totalHits.value()); } } } } - // Test to cover intersectRight with CELL_CROSSES_QUERY case and ensure comprehensive coverage for intersectRight + // Test to cover intersectRight with CELL_CROSSES_QUERY case public void testIntersectRightCellCrossesQuery() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { int dims = 1; - long[] scratch = new long[dims]; - // Create documents that will result in cells that cross the query boundary - for (int i = 0; i < 1000; i++) { - scratch[0] = i; - iw.addDocument(asList(new LongPoint("point", scratch[0]), new NumericDocValuesField("point", scratch[0]))); + int numPoints = RandomNumbers.randomIntBetween(random(), 1000, 3000); + for (int i = 0; i < numPoints; i++) { + Document doc = new Document(); + numericType.addField(doc, numericType.fieldName, i); + numericType.addDocValuesField(doc, numericType.fieldName, i); + iw.addDocument(doc); + if (random().nextInt(100) == 0) { + iw.flush(); + } } iw.flush(); - iw.forceMerge(1); - + if (random().nextBoolean()) { + iw.forceMerge(1); + } try (IndexReader reader = iw.getReader()) { // Query that will partially overlap with tree nodes (CELL_CROSSES_QUERY) - // This range will intersect with some tree nodes but not completely contain them - long lower = 250; - long upper = 750; - final int size = 100; + long lower = RandomNumbers.randomLongBetween(random(), numPoints / 4, numPoints / 2); + long upper = RandomNumbers.randomLongBetween(random(), numPoints / 2, 3 * numPoints / 4); + final int size = RandomNumbers.randomIntBetween(random(), 50, 200); ApproximatePointRangeQuery query = new ApproximatePointRangeQuery( - "point", - pack(lower).bytes, - pack(upper).bytes, + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), dims, - 100, + size, SortOrder.DESC, - ApproximatePointRangeQuery.LONG_FORMAT + numericType.format ); - IndexSearcher searcher = new IndexSearcher(reader); - Sort sort = new Sort(new SortField("point", SortField.Type.LONG, true)); // DESC + Sort sort = new Sort(new SortField(numericType.getSortFieldName(), numericType.getSortFieldType(), true)); TopDocs topDocs = searcher.search(query, size, sort); - - assertEquals("Should return exactly size value documents", size, topDocs.scoreDocs.length); - // For Desc sort the ApproximatePointRangeQuery will slightly over collect to retain the highest matched docs - assertTrue("Should collect at least requested number of documents", topDocs.totalHits.value() >= 100); + long expectedHits = upper - lower + 1; + int expectedReturnSize = (int) Math.min(size, expectedHits); + assertEquals("Should return min(size, hits) documents", expectedReturnSize, topDocs.scoreDocs.length); + assertTrue("Should collect at least min(size, hits) documents", topDocs.totalHits.value() >= expectedReturnSize); } } } @@ -618,67 +834,72 @@ public void testIntersectRightSingleChildNode() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { int dims = 1; - long[] scratch = new long[dims]; - - for (int i = 0; i < 100; i++) { - scratch[0] = 1000L; - iw.addDocument(asList(new LongPoint("point", scratch[0]), new NumericDocValuesField("point", scratch[0]))); + int numSameValueDocs = RandomNumbers.randomIntBetween(random(), 1000, 3000); + long sameValue = RandomNumbers.randomLongBetween(random(), 500, 2000); + for (int i = 0; i < numSameValueDocs; i++) { + Document doc = new Document(); + numericType.addField(doc, numericType.fieldName, sameValue); + numericType.addDocValuesField(doc, numericType.fieldName, sameValue); + iw.addDocument(doc); } - scratch[0] = 987654321L; - iw.addDocument(asList(new LongPoint("point", scratch[0]), new NumericDocValuesField("point", scratch[0]))); - + long highValue = RandomNumbers.randomLongBetween(random(), 900000000L, 999999999L); + Document doc = new Document(); + numericType.addField(doc, numericType.fieldName, highValue); + numericType.addDocValuesField(doc, numericType.fieldName, highValue); + iw.addDocument(doc); iw.flush(); - iw.forceMerge(1); - + if (random().nextBoolean()) { + iw.forceMerge(1); + } try (IndexReader reader = iw.getReader()) { - long lower = 500L; - long upper = 999999999L; - final int size = 50; + long lower = RandomNumbers.randomLongBetween(random(), 0, sameValue - 100); + long upper = RandomNumbers.randomLongBetween(random(), highValue, highValue + 1000000L); + final int size = RandomNumbers.randomIntBetween(random(), 20, 100); ApproximatePointRangeQuery query = new ApproximatePointRangeQuery( - "point", - pack(lower).bytes, - pack(upper).bytes, + numericType.fieldName, + numericType.encode(lower), + numericType.encode(upper), dims, size, SortOrder.DESC, - ApproximatePointRangeQuery.LONG_FORMAT + numericType.format ); - IndexSearcher searcher = new IndexSearcher(reader); - Sort sort = new Sort(new SortField("point", SortField.Type.LONG, true)); + Sort sort = new Sort(new SortField(numericType.getSortFieldName(), numericType.getSortFieldType(), true)); TopDocs topDocs = searcher.search(query, size, sort); - - assertEquals("Should return exactly size value documents", size, topDocs.scoreDocs.length); + int totalDocs = numSameValueDocs + 1; + int expectedReturnSize = Math.min(size, totalDocs); + assertEquals("Should return min(size, totalDocs) documents", expectedReturnSize, topDocs.scoreDocs.length); } } } } - // Following test replicates the http_logs dataset public void testHttpLogTimestampDistribution() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { int dims = 1; + String fieldName = numericType.fieldName; // Sparse range: 100-199 (100 docs, one per value) for (int i = 100; i < 200; i++) { Document doc = new Document(); - doc.add(new LongPoint("timestamp", i)); - doc.add(new NumericDocValuesField("timestamp", i)); + numericType.addField(doc, fieldName, i); + numericType.addDocValuesField(doc, fieldName, i); iw.addDocument(doc); } // Dense range: 1000-1999 (5000 docs, 5 per value) for (int i = 0; i < 5000; i++) { long value = 1000 + (i / 5); // Creates 5 docs per value from 1000-1999 Document doc = new Document(); - doc.add(new LongPoint("timestamp", value)); - doc.add(new NumericDocValuesField("timestamp", value)); + numericType.addField(doc, fieldName, value); + numericType.addDocValuesField(doc, fieldName, value); iw.addDocument(doc); } // 0-99 (100 docs) for (int i = 0; i < 100; i++) { Document doc = new Document(); - doc.add(new LongPoint("timestamp", i)); - doc.add(new NumericDocValuesField("timestamp", i)); + numericType.addField(doc, fieldName, i); + numericType.addDocValuesField(doc, fieldName, i); iw.addDocument(doc); } iw.flush(); @@ -686,11 +907,11 @@ public void testHttpLogTimestampDistribution() throws IOException { try (IndexReader reader = iw.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); // Test sparse region - testApproximateVsExactQuery(searcher, "timestamp", 100, 199, 50, dims); + testApproximateVsExactQuery(searcher, fieldName, 100, 199, 50, dims); // Test dense region - testApproximateVsExactQuery(searcher, "timestamp", 1000, 1500, 100, dims); + testApproximateVsExactQuery(searcher, fieldName, 1000, 1500, 100, dims); // Test across regions - testApproximateVsExactQuery(searcher, "timestamp", 0, 2000, 200, dims); + testApproximateVsExactQuery(searcher, fieldName, 0, 2000, 200, dims); } } } @@ -701,30 +922,41 @@ public void testNycTaxiDataDistribution() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { int dims = 1; + String fieldName = numericType.fieldName; // Create NYC taxi fare distribution with different ranges + // Low fares: 250-500 (sparse) for (long fare = 250; fare <= 500; fare++) { - iw.addDocument(asList(new LongPoint("fare_amount", fare), new NumericDocValuesField("fare_amount", fare))); + Document doc = new Document(); + numericType.addField(doc, fieldName, fare); + numericType.addDocValuesField(doc, fieldName, fare); + iw.addDocument(doc); } // Typical fares: 1000-3000 (dense, 5 docs per value) for (long fare = 1000; fare <= 3000; fare++) { for (int dup = 0; dup < 5; dup++) { - iw.addDocument(asList(new LongPoint("fare_amount", fare), new NumericDocValuesField("fare_amount", fare))); + Document doc = new Document(); + numericType.addField(doc, fieldName, fare); + numericType.addDocValuesField(doc, fieldName, fare); + iw.addDocument(doc); } } // High fares: 10000-20000 (sparse, 1 doc every 100) for (long fare = 10000; fare <= 20000; fare += 100) { - iw.addDocument(asList(new LongPoint("fare_amount", fare), new NumericDocValuesField("fare_amount", fare))); + Document doc = new Document(); + numericType.addField(doc, fieldName, fare); + numericType.addDocValuesField(doc, fieldName, fare); + iw.addDocument(doc); } iw.flush(); iw.forceMerge(1); try (IndexReader reader = iw.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); // Test 1: Query for typical fare range - testApproximateVsExactQuery(searcher, "fare_amount", 1000, 3000, 100, dims); + testApproximateVsExactQuery(searcher, fieldName, 1000, 3000, 100, dims); // Test 2: Query for high fare range - testApproximateVsExactQuery(searcher, "fare_amount", 10000, 20000, 50, dims); + testApproximateVsExactQuery(searcher, fieldName, 10000, 20000, 50, dims); // Test 3: Query for low fares - testApproximateVsExactQuery(searcher, "fare_amount", 250, 500, 50, dims); + testApproximateVsExactQuery(searcher, fieldName, 250, 500, 50, dims); } } } @@ -735,15 +967,15 @@ private void testApproximateVsExactQuery(IndexSearcher searcher, String field, l // Test with approximate query ApproximatePointRangeQuery approxQuery = new ApproximatePointRangeQuery( field, - pack(lower).bytes, - pack(upper).bytes, + numericType.encode(lower), + numericType.encode(upper), dims, size, null, - ApproximatePointRangeQuery.LONG_FORMAT + numericType.format ); // Test with exact query - Query exactQuery = LongPoint.newRangeQuery(field, lower, upper); + Query exactQuery = numericType.rangeQuery(field, lower, upper); TopDocs approxDocs = searcher.search(approxQuery, size); TopDocs exactDocs = searcher.search(exactQuery, size); // Verify approximate query returns correct number of results @@ -757,24 +989,26 @@ private void testApproximateVsExactQuery(IndexSearcher searcher, String field, l ); } // Test with sorting (ASC and DESC) - Sort ascSort = new Sort(new SortField(field, SortField.Type.LONG)); - Sort descSort = new Sort(new SortField(field, SortField.Type.LONG, true)); + Sort ascSort = new Sort(new SortField(numericType.getSortFieldName(), numericType.getSortFieldType())); + Sort descSort = new Sort(new SortField(numericType.getSortFieldName(), numericType.getSortFieldType(), true)); // Test ASC sort ApproximatePointRangeQuery approxQueryAsc = new ApproximatePointRangeQuery( field, - pack(lower).bytes, - pack(upper).bytes, + numericType.encode(lower), + numericType.encode(upper), dims, size, SortOrder.ASC, - ApproximatePointRangeQuery.LONG_FORMAT + numericType.format ); TopDocs approxDocsAsc = searcher.search(approxQueryAsc, size, ascSort); TopDocs exactDocsAsc = searcher.search(exactQuery, size, ascSort); // Verify results match - for (int i = 0; i < size; i++) { + int compareSize = Math.min(size, Math.min(approxDocsAsc.scoreDocs.length, exactDocsAsc.scoreDocs.length)); + for (int i = 0; i < compareSize; i++) { assertEquals("ASC sorted results should match at position " + i, exactDocsAsc.scoreDocs[i].doc, approxDocsAsc.scoreDocs[i].doc); } + assertEquals("Should return same number of documents", exactDocsAsc.scoreDocs.length, approxDocsAsc.scoreDocs.length); assertEquals("Should return exactly size value documents", size, approxDocsAsc.scoreDocs.length); assertEquals( "Should return exactly size value documents as regular query", @@ -784,23 +1018,25 @@ private void testApproximateVsExactQuery(IndexSearcher searcher, String field, l // Test DESC sort ApproximatePointRangeQuery approxQueryDesc = new ApproximatePointRangeQuery( field, - pack(lower).bytes, - pack(upper).bytes, + numericType.encode(lower), + numericType.encode(upper), dims, size, SortOrder.DESC, - ApproximatePointRangeQuery.LONG_FORMAT + numericType.format ); TopDocs approxDocsDesc = searcher.search(approxQueryDesc, size, descSort); TopDocs exactDocsDesc = searcher.search(exactQuery, size, descSort); // Verify the results match - for (int i = 0; i < size; i++) { + compareSize = Math.min(size, Math.min(approxDocsDesc.scoreDocs.length, exactDocsDesc.scoreDocs.length)); + for (int i = 0; i < compareSize; i++) { assertEquals( "DESC sorted results should match at position " + i, exactDocsDesc.scoreDocs[i].doc, approxDocsDesc.scoreDocs[i].doc ); } + assertEquals("Should return same number of documents", exactDocsDesc.scoreDocs.length, approxDocsDesc.scoreDocs.length); assertEquals("Should return exactly size value documents", size, approxDocsAsc.scoreDocs.length); assertEquals( "Should return exactly size value documents as regular query", From 54b8f05fbae6768c9015ce14c07b14d5f8527090 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Fri, 4 Jul 2025 03:42:51 +0200 Subject: [PATCH 17/51] Bump BouncyCastle and password4j (#18668) Changes: - Bump org.bouncycastle:bctls-fips from 2.0.19 to 2.0.20 - Bump org.bouncycastle:bcpkix-fips from 2.0.7 to 2.0.8 - Bump org.bouncycastle:bcpg-fips from 2.0.10 to 2.0.11 - Bump com.password4j:password4j from 1.8.2 to 1.8.3 Signed-off-by: Andrey Pleskach --- CHANGELOG.md | 4 ++++ client/rest/licenses/bctls-fips-2.0.19.jar.sha1 | 1 - client/rest/licenses/bctls-fips-2.0.20.jar.sha1 | 1 + .../tools/plugin-cli/licenses/bcpg-fips-2.0.10.jar.sha1 | 1 - .../tools/plugin-cli/licenses/bcpg-fips-2.0.11.jar.sha1 | 1 + gradle/libs.versions.toml | 9 +++++---- libs/ssl-config/licenses/bcpkix-fips-2.0.7.jar.sha1 | 1 - libs/ssl-config/licenses/bcpkix-fips-2.0.8.jar.sha1 | 1 + libs/ssl-config/licenses/bctls-fips-2.0.19.jar.sha1 | 1 - libs/ssl-config/licenses/bctls-fips-2.0.20.jar.sha1 | 1 + .../identity-shiro/licenses/password4j-1.8.2.jar.sha1 | 1 - .../identity-shiro/licenses/password4j-1.8.3.jar.sha1 | 1 + test/framework/licenses/bcpkix-fips-2.0.7.jar.sha1 | 1 - test/framework/licenses/bcpkix-fips-2.0.8.jar.sha1 | 1 + 14 files changed, 15 insertions(+), 10 deletions(-) delete mode 100644 client/rest/licenses/bctls-fips-2.0.19.jar.sha1 create mode 100644 client/rest/licenses/bctls-fips-2.0.20.jar.sha1 delete mode 100644 distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.10.jar.sha1 create mode 100644 distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.11.jar.sha1 delete mode 100644 libs/ssl-config/licenses/bcpkix-fips-2.0.7.jar.sha1 create mode 100644 libs/ssl-config/licenses/bcpkix-fips-2.0.8.jar.sha1 delete mode 100644 libs/ssl-config/licenses/bctls-fips-2.0.19.jar.sha1 create mode 100644 libs/ssl-config/licenses/bctls-fips-2.0.20.jar.sha1 delete mode 100644 plugins/identity-shiro/licenses/password4j-1.8.2.jar.sha1 create mode 100644 plugins/identity-shiro/licenses/password4j-1.8.3.jar.sha1 delete mode 100644 test/framework/licenses/bcpkix-fips-2.0.7.jar.sha1 create mode 100644 test/framework/licenses/bcpkix-fips-2.0.8.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 194fd74c63be4..f98499607790c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.squareup.okio:okio` from 3.13.0 to 3.14.0 ([#18645](https://github.com/opensearch-project/OpenSearch/pull/18645)) - Bump `com.netflix.nebula.ospackage-base` from 11.11.2 to 12.0.0 ([#18646](https://github.com/opensearch-project/OpenSearch/pull/18646)) - Bump `com.azure:azure-storage-blob` from 12.30.0 to 12.30.1 ([#18644](https://github.com/opensearch-project/OpenSearch/pull/18644)) +- Bump `org.bouncycastle:bctls-fips` from 2.0.19 to 2.0.20 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) +- Bump `org.bouncycastle:bcpkix-fips` from 2.0.7 to 2.0.8 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) +- Bump `org.bouncycastle:bcpg-fips` from 2.0.10 to 2.0.11 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) +- Bump `com.password4j:password4j` from 1.8.2 to 1.8.3 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) ### Deprecated diff --git a/client/rest/licenses/bctls-fips-2.0.19.jar.sha1 b/client/rest/licenses/bctls-fips-2.0.19.jar.sha1 deleted file mode 100644 index 387635e9e1594..0000000000000 --- a/client/rest/licenses/bctls-fips-2.0.19.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9cc33650ede63bc1a8281ed5c8e1da314d50bc76 \ No newline at end of file diff --git a/client/rest/licenses/bctls-fips-2.0.20.jar.sha1 b/client/rest/licenses/bctls-fips-2.0.20.jar.sha1 new file mode 100644 index 0000000000000..66cd82b49b537 --- /dev/null +++ b/client/rest/licenses/bctls-fips-2.0.20.jar.sha1 @@ -0,0 +1 @@ +1138f7896e0d1bb0d924bc868ed2dfda4f69470e \ No newline at end of file diff --git a/distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.10.jar.sha1 b/distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.10.jar.sha1 deleted file mode 100644 index c7aa41c7996d8..0000000000000 --- a/distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f21aff3416359ad20b2712c0727696858a2e769a \ No newline at end of file diff --git a/distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.11.jar.sha1 b/distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.11.jar.sha1 new file mode 100644 index 0000000000000..39805c3a32614 --- /dev/null +++ b/distribution/tools/plugin-cli/licenses/bcpg-fips-2.0.11.jar.sha1 @@ -0,0 +1 @@ +19f38a0d8048e87039b1bb6c1ba4d2b104891d04 \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d8bf39525abfe..fe0716b4b62f6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -62,11 +62,12 @@ hadoop3 = "3.3.6" # - plugins/ingest-attachment (transitive dependency, check the upstream POM) # - distribution/tools/plugin-cli bouncycastle_jce = "2.0.0" -bouncycastle_tls = "2.0.19" -bouncycastle_pkix = "2.0.7" -bouncycastle_pg = "2.0.10" +bouncycastle_tls = "2.0.20" +bouncycastle_pkix = "2.0.8" +bouncycastle_pg = "2.0.11" bouncycastle_util = "2.0.3" -password4j = "1.8.2" +password4j = "1.8.3" + # test dependencies randomizedrunner = "2.7.1" junit = "4.13.2" diff --git a/libs/ssl-config/licenses/bcpkix-fips-2.0.7.jar.sha1 b/libs/ssl-config/licenses/bcpkix-fips-2.0.7.jar.sha1 deleted file mode 100644 index 5df930b54fe44..0000000000000 --- a/libs/ssl-config/licenses/bcpkix-fips-2.0.7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -01eea0f325315ca6295b0a6926ff862d8001cdf9 \ No newline at end of file diff --git a/libs/ssl-config/licenses/bcpkix-fips-2.0.8.jar.sha1 b/libs/ssl-config/licenses/bcpkix-fips-2.0.8.jar.sha1 new file mode 100644 index 0000000000000..69293a600d472 --- /dev/null +++ b/libs/ssl-config/licenses/bcpkix-fips-2.0.8.jar.sha1 @@ -0,0 +1 @@ +aad7b0fcf55892e7ff7e2d23a290f143f4bb56e0 \ No newline at end of file diff --git a/libs/ssl-config/licenses/bctls-fips-2.0.19.jar.sha1 b/libs/ssl-config/licenses/bctls-fips-2.0.19.jar.sha1 deleted file mode 100644 index 387635e9e1594..0000000000000 --- a/libs/ssl-config/licenses/bctls-fips-2.0.19.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9cc33650ede63bc1a8281ed5c8e1da314d50bc76 \ No newline at end of file diff --git a/libs/ssl-config/licenses/bctls-fips-2.0.20.jar.sha1 b/libs/ssl-config/licenses/bctls-fips-2.0.20.jar.sha1 new file mode 100644 index 0000000000000..66cd82b49b537 --- /dev/null +++ b/libs/ssl-config/licenses/bctls-fips-2.0.20.jar.sha1 @@ -0,0 +1 @@ +1138f7896e0d1bb0d924bc868ed2dfda4f69470e \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/password4j-1.8.2.jar.sha1 b/plugins/identity-shiro/licenses/password4j-1.8.2.jar.sha1 deleted file mode 100644 index bee14467d32a2..0000000000000 --- a/plugins/identity-shiro/licenses/password4j-1.8.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f8ac106c667c0b081075e81a90dc92861b9bb66e \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/password4j-1.8.3.jar.sha1 b/plugins/identity-shiro/licenses/password4j-1.8.3.jar.sha1 new file mode 100644 index 0000000000000..fc1e6d2e1b2a8 --- /dev/null +++ b/plugins/identity-shiro/licenses/password4j-1.8.3.jar.sha1 @@ -0,0 +1 @@ +619cb40bd02455cf2c1a858bfaf5726d79acdf0c \ No newline at end of file diff --git a/test/framework/licenses/bcpkix-fips-2.0.7.jar.sha1 b/test/framework/licenses/bcpkix-fips-2.0.7.jar.sha1 deleted file mode 100644 index 5df930b54fe44..0000000000000 --- a/test/framework/licenses/bcpkix-fips-2.0.7.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -01eea0f325315ca6295b0a6926ff862d8001cdf9 \ No newline at end of file diff --git a/test/framework/licenses/bcpkix-fips-2.0.8.jar.sha1 b/test/framework/licenses/bcpkix-fips-2.0.8.jar.sha1 new file mode 100644 index 0000000000000..69293a600d472 --- /dev/null +++ b/test/framework/licenses/bcpkix-fips-2.0.8.jar.sha1 @@ -0,0 +1 @@ +aad7b0fcf55892e7ff7e2d23a290f143f4bb56e0 \ No newline at end of file From fbfa6265612848e8ec2fad50de692b7aa800281c Mon Sep 17 00:00:00 2001 From: Gagan6164 <32809202+Gagan6164@users.noreply.github.com> Date: Fri, 4 Jul 2025 11:24:43 +0530 Subject: [PATCH 18/51] Add support for custom index name resolver from cluster plugin (#18593) Signed-off-by: Gagan Singh Saini --- CHANGELOG.md | 1 + .../WarmIndexSegmentReplicationIT.java | 1 + .../org/opensearch/cluster/ClusterModule.java | 18 +++++- .../metadata/IndexNameExpressionResolver.java | 42 ++++++++++++-- .../org/opensearch/plugins/ClusterPlugin.java | 9 +++ .../cluster/ClusterModuleTests.java | 56 +++++++++++++++++++ .../IndexNameExpressionResolverTests.java | 15 +++++ 7 files changed, 137 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f98499607790c..0babaf80b126b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 3.x] ### Added - Add support for Warm Indices Write Block on Flood Watermark breach ([#18375](https://github.com/opensearch-project/OpenSearch/pull/18375)) +- Add support for custom index name resolver from cluster plugin ([#18593](https://github.com/opensearch-project/OpenSearch/pull/18593)) - Ability to run Code Coverage with Gradle and produce the jacoco reports locally ([#18509](https://github.com/opensearch-project/OpenSearch/issues/18509)) - Add NodeResourceUsageStats to ClusterInfo ([#18480](https://github.com/opensearch-project/OpenSearch/issues/18472)) - Introduce SecureHttpTransportParameters experimental API (to complement SecureTransportParameters counterpart) ([#18572](https://github.com/opensearch-project/OpenSearch/issues/18572)) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/WarmIndexSegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/WarmIndexSegmentReplicationIT.java index 2f0b1e5b07f16..91754f069217c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/WarmIndexSegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/WarmIndexSegmentReplicationIT.java @@ -517,6 +517,7 @@ public void testReplicationPostDeleteAndForceMerge() throws Exception { assertHitCount(client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount + 1); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/18157") public void testScrollWithConcurrentIndexAndSearch() throws Exception { final String primary = internalCluster().startDataAndWarmNodes(1).get(0); final String replica = internalCluster().startDataAndWarmNodes(1).get(0); diff --git a/server/src/main/java/org/opensearch/cluster/ClusterModule.java b/server/src/main/java/org/opensearch/cluster/ClusterModule.java index fc7dae7854ff2..7dbc0d0ee88e3 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterModule.java @@ -41,6 +41,7 @@ import org.opensearch.cluster.metadata.DataStreamMetadata; import org.opensearch.cluster.metadata.IndexGraveyard; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver.ExpressionResolver; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.MetadataDeleteIndexService; import org.opensearch.cluster.metadata.MetadataIndexAliasesService; @@ -158,7 +159,7 @@ public ClusterModule( this.allocationDeciders = new AllocationDeciders(deciderList); this.shardsAllocator = createShardsAllocator(settings, clusterService.getClusterSettings(), clusterPlugins); this.clusterService = clusterService; - this.indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext); + this.indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, getCustomResolvers(clusterPlugins)); this.allocationService = new AllocationService( allocationDeciders, shardsAllocator, @@ -439,6 +440,21 @@ private static ShardsAllocator createShardsAllocator( return Objects.requireNonNull(allocatorSupplier.get(), "ShardsAllocator factory for [" + allocatorName + "] returned null"); } + private static List getCustomResolvers(List clusterPlugins) { + Map resolvers = new HashMap<>(); + clusterPlugins.stream().flatMap(c -> c.getIndexNameCustomResolvers().stream()).forEach(r -> addCustomResolver(resolvers, r)); + return Collections.unmodifiableList(new ArrayList<>(resolvers.values())); + } + + private static void addCustomResolver( + Map resolvers, + IndexNameExpressionResolver.ExpressionResolver customResolver + ) { + if (resolvers.put(customResolver.getClass(), customResolver) != null) { + throw new IllegalArgumentException("Cannot specify expression resolver [" + customResolver.getClass().getName() + "] twice"); + } + } + public AllocationService getAllocationService() { return allocationService; } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java index 24ff83d638d4b..abfabdd10c340 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java @@ -88,14 +88,34 @@ public class IndexNameExpressionResolver { private final DateMathExpressionResolver dateMathExpressionResolver = new DateMathExpressionResolver(); private final WildcardExpressionResolver wildcardExpressionResolver = new WildcardExpressionResolver(); - private final List expressionResolvers = List.of(dateMathExpressionResolver, wildcardExpressionResolver); + private final List expressionResolvers; + private final List customResolvers = new ArrayList<>(); private final ThreadContext threadContext; public IndexNameExpressionResolver(ThreadContext threadContext) { + expressionResolvers = new ArrayList<>(); + expressionResolvers.add(dateMathExpressionResolver); + expressionResolvers.add(wildcardExpressionResolver); this.threadContext = Objects.requireNonNull(threadContext, "Thread Context must not be null"); } + public IndexNameExpressionResolver(ThreadContext threadContext, List resolvers) { + // Set custom resolvers to the top of the list to ensure to WildcardExpressionResolver is triggered at the end, + // otherwise it will throw exception in case it cannot resolve any keywords which are expected to be handled + // by custom resolvers. + customResolvers.addAll(resolvers); + expressionResolvers = new ArrayList<>(customResolvers); + expressionResolvers.add(dateMathExpressionResolver); + expressionResolvers.add(wildcardExpressionResolver); + this.threadContext = Objects.requireNonNull(threadContext, "Thread Context must not be null"); + } + + // Visible for testing + public List getExpressionResolvers() { + return new ArrayList<>(this.expressionResolvers); + } + /** * Same as {@link #concreteIndexNames(ClusterState, IndicesOptions, String...)}, but the index expressions and options * are encapsulated in the specified request. @@ -171,7 +191,14 @@ public List dataStreamNames(ClusterState state, IndicesOptions options, indexExpressions = new String[] { "*" }; } - List dataStreams = wildcardExpressionResolver.resolve(context, Arrays.asList(indexExpressions)); + // Using customResolvers to filter out invalid expressions + List finalExpressions = Arrays.asList(indexExpressions); + for (ExpressionResolver resolver : customResolvers) { + finalExpressions = resolver.resolve(context, finalExpressions); + } + + List dataStreams = wildcardExpressionResolver.resolve(context, finalExpressions); + return ((dataStreams == null) ? List.of() : dataStreams).stream() .map(x -> state.metadata().getIndicesLookup().get(x)) .filter(Objects::nonNull) @@ -778,6 +805,7 @@ public boolean isSystemIndexAccessAllowed() { * * @opensearch.internal */ + @PublicApi(since = "3.1.0") public static class Context { private final ClusterState state; @@ -789,7 +817,7 @@ public static class Context { private final boolean preserveDataStreams; private final boolean isSystemIndexAccessAllowed; - Context(ClusterState state, IndicesOptions options, boolean isSystemIndexAccessAllowed) { + public Context(ClusterState state, IndicesOptions options, boolean isSystemIndexAccessAllowed) { this(state, options, System.currentTimeMillis(), isSystemIndexAccessAllowed); } @@ -903,7 +931,13 @@ public boolean isSystemIndexAccessAllowed() { } } - private interface ExpressionResolver { + /** + * Expression resolver for index name expressions. + * + * @opensearch.internal + */ + @PublicApi(since = "3.1.0") + public interface ExpressionResolver { /** * Resolves the list of expressions into other expressions if possible (possible concrete indices and aliases, but diff --git a/server/src/main/java/org/opensearch/plugins/ClusterPlugin.java b/server/src/main/java/org/opensearch/plugins/ClusterPlugin.java index 1edd9f52d97a7..9b2ddd0bed41a 100644 --- a/server/src/main/java/org/opensearch/plugins/ClusterPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/ClusterPlugin.java @@ -32,6 +32,7 @@ package org.opensearch.plugins; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.allocation.ExistingShardsAllocator; import org.opensearch.cluster.routing.allocation.allocator.ShardsAllocator; @@ -85,6 +86,14 @@ default Map getExistingShardsAllocators() { return Collections.emptyMap(); } + /** + * Returns List of custom index name resolvers which can support additional custom wildcards. + * @return List of {@link IndexNameExpressionResolver.ExpressionResolver} + */ + default Collection getIndexNameCustomResolvers() { + return Collections.emptyList(); + } + /** * Called when the node is started * diff --git a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java index c620b03042007..82551bc26e224 100644 --- a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java +++ b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java @@ -32,6 +32,7 @@ package org.opensearch.cluster; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.WorkloadGroupMetadata; @@ -127,6 +128,61 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing } } + static class FakeExpressionResolver implements IndexNameExpressionResolver.ExpressionResolver { + @Override + public List resolve(IndexNameExpressionResolver.Context context, List expressions) { + throw new UnsupportedOperationException("resolve operation not supported on FakeExpressionResolver"); + } + } + + static class AnotherFakeExpressionResolver implements IndexNameExpressionResolver.ExpressionResolver { + @Override + public List resolve(IndexNameExpressionResolver.Context context, List expressions) { + throw new UnsupportedOperationException("resolve operation not supported on FakeExpressionResolver"); + } + } + + public void testRegisterCustomExpressionResolver() { + FakeExpressionResolver customResolver1 = new FakeExpressionResolver(); + AnotherFakeExpressionResolver customResolver2 = new AnotherFakeExpressionResolver(); + List clusterPlugins = Collections.singletonList(new ClusterPlugin() { + @Override + public Collection getIndexNameCustomResolvers() { + return Arrays.asList(customResolver1, customResolver2); + } + }); + ClusterModule module = new ClusterModule( + Settings.EMPTY, + clusterService, + clusterPlugins, + clusterInfoService, + null, + threadContext, + null + ); + assertTrue(module.getIndexNameExpressionResolver().getExpressionResolvers().contains(customResolver1)); + assertTrue(module.getIndexNameExpressionResolver().getExpressionResolvers().contains(customResolver2)); + } + + public void testRegisterCustomExpressionResolverDuplicate() { + FakeExpressionResolver customResolver1 = new FakeExpressionResolver(); + FakeExpressionResolver customResolver2 = new FakeExpressionResolver(); + List clusterPlugins = Collections.singletonList(new ClusterPlugin() { + @Override + public Collection getIndexNameCustomResolvers() { + return Arrays.asList(customResolver1, customResolver2); + } + }); + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> new ClusterModule(Settings.EMPTY, clusterService, clusterPlugins, clusterInfoService, null, threadContext, null) + ); + assertEquals( + "Cannot specify expression resolver [org.opensearch.cluster.ClusterModuleTests$FakeExpressionResolver] twice", + ex.getMessage() + ); + } + public void testRegisterClusterDynamicSettingDuplicate() { try { new SettingsModule(Settings.EMPTY, EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java index fda2f411b1994..c2caae1dc2586 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -45,6 +45,7 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata.State; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver.ExpressionResolver; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.Strings; @@ -2520,6 +2521,20 @@ public void testDataStreamsNames() { assertThat(names, empty()); } + static class FakeExpressionResolver implements IndexNameExpressionResolver.ExpressionResolver { + @Override + public List resolve(IndexNameExpressionResolver.Context context, List expressions) { + throw new UnsupportedOperationException("resolve operation not supported on FakeExpressionResolver"); + } + } + + public void testAddCustomResolver() { + FakeExpressionResolver customResolver = new FakeExpressionResolver(); + List customResolvers = Collections.singletonList(customResolver); + IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, customResolvers); + assertEquals(indexNameExpressionResolver.getExpressionResolvers().get(0), customResolver); + } + private ClusterState systemIndexTestClusterState() { Settings settings = Settings.builder().build(); Metadata.Builder mdBuilder = Metadata.builder() From 5d9695cc341f95c06873497252761c8cf64e7960 Mon Sep 17 00:00:00 2001 From: rayshrey <121871912+rayshrey@users.noreply.github.com> Date: Fri, 4 Jul 2025 11:30:06 +0530 Subject: [PATCH 19/51] Remove clone holder and add Cleaner logic to clean up clones in FullFileCachedIndexInput (#18662) Signed-off-by: Shreyansh Ray --- .../filecache/FullFileCachedIndexInput.java | 47 ++++++++++------ .../remote/filecache/FileCacheTests.java | 4 ++ .../filecache/FileCachedIndexInputTests.java | 4 ++ .../FullFileCachedIndexInputTests.java | 55 ++++++++++++++----- 4 files changed, 80 insertions(+), 30 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInput.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInput.java index 9383c53d6d830..deb8f437bfd63 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInput.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInput.java @@ -13,23 +13,23 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.IndexInput; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; import java.io.IOException; +import java.lang.ref.Cleaner; import java.nio.file.Path; -import java.util.HashSet; -import java.util.Set; /** * Extension of {@link FileCachedIndexInput} for full files for handling clones and slices - * We maintain a clone map so that we can close them when the parent IndexInput is closed so that ref count is properly maintained in file cache - * Closing of clones explicitly is needed as Lucene does not guarantee that it will close the clones + * Since Lucene does not guarantee that it will close the clones/slices, we have created a Cleaner which handles closing of the clones/slices when they become phantom reachable * https://github.com/apache/lucene/blob/8340b01c3cc229f33584ce2178b07b8984daa6a9/lucene/core/src/java/org/apache/lucene/store/IndexInput.java#L32-L33 * @opensearch.experimental */ @ExperimentalApi public class FullFileCachedIndexInput extends FileCachedIndexInput { private static final Logger logger = LogManager.getLogger(FullFileCachedIndexInput.class); - private final Set clones; + private final IndexInputHolder indexInputHolder; + private static final Cleaner CLEANER = Cleaner.create(OpenSearchExecutors.daemonThreadFactory("index-input-cleaner")); public FullFileCachedIndexInput(FileCache cache, Path filePath, IndexInput underlyingIndexInput) { this(cache, filePath, underlyingIndexInput, false); @@ -37,7 +37,8 @@ public FullFileCachedIndexInput(FileCache cache, Path filePath, IndexInput under public FullFileCachedIndexInput(FileCache cache, Path filePath, IndexInput underlyingIndexInput, boolean isClone) { super(cache, filePath, underlyingIndexInput, isClone); - clones = new HashSet<>(); + indexInputHolder = new IndexInputHolder(underlyingIndexInput, isClone, cache, filePath); + CLEANER.register(this, indexInputHolder); } /** @@ -48,7 +49,6 @@ public FullFileCachedIndexInput(FileCache cache, Path filePath, IndexInput under public FullFileCachedIndexInput clone() { FullFileCachedIndexInput clonedIndexInput = new FullFileCachedIndexInput(cache, filePath, luceneIndexInput.clone(), true); cache.incRef(filePath); - clones.add(clonedIndexInput); return clonedIndexInput; } @@ -74,7 +74,6 @@ public IndexInput slice(String sliceDescription, long offset, long length) throw } IndexInput slicedLuceneIndexInput = luceneIndexInput.slice(sliceDescription, offset, length); FullFileCachedIndexInput slicedIndexInput = new FullFileCachedIndexInput(cache, filePath, slicedLuceneIndexInput, true); - clones.add(slicedIndexInput); cache.incRef(filePath); return slicedIndexInput; } @@ -88,21 +87,37 @@ public void close() throws IOException { if (isClone) { cache.decRef(filePath); } - clones.forEach(indexInput -> { - try { - indexInput.close(); - } catch (Exception e) { - logger.trace("Exception while closing clone - {}", e.getMessage()); - } - }); try { luceneIndexInput.close(); } catch (AlreadyClosedException e) { logger.trace("FullFileCachedIndexInput already closed"); } luceneIndexInput = null; - clones.clear(); closed = true; } } + + private static class IndexInputHolder implements Runnable { + private final IndexInput indexInput; + private final FileCache cache; + private final boolean isClone; + private final Path path; + + IndexInputHolder(IndexInput indexInput, boolean isClone, FileCache cache, Path path) { + this.indexInput = indexInput; + this.isClone = isClone; + this.cache = cache; + this.path = path; + } + + @Override + public void run() { + try { + indexInput.close(); + if (isClone) cache.decRef(path); + } catch (IOException e) { + logger.error("Failed to close IndexInput while clearing phantom reachable object"); + } + } + } } diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java index 643caa85b5862..3ab193c5325c6 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java @@ -8,6 +8,8 @@ package org.opensearch.index.store.remote.filecache; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -18,6 +20,7 @@ import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.store.remote.directory.RemoteSnapshotDirectoryFactory; +import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter; import org.opensearch.index.store.remote.utils.FileTypeUtils; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; @@ -28,6 +31,7 @@ import java.nio.file.attribute.PosixFilePermissions; import java.util.List; +@ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class) public class FileCacheTests extends OpenSearchTestCase { // need concurrency level to be static to make these tests more deterministic because capacity per segment is dependent on // (total capacity) / (concurrency level) so having high concurrency level might trigger early evictions which is tolerable in real life diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCachedIndexInputTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCachedIndexInputTests.java index ce0a4d7bf3c02..41e76d0b762ea 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCachedIndexInputTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCachedIndexInputTests.java @@ -8,18 +8,22 @@ package org.opensearch.index.store.remote.filecache; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.breaker.NoopCircuitBreaker; +import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; import java.io.IOException; import java.nio.file.Path; +@ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class) public class FileCachedIndexInputTests extends OpenSearchTestCase { protected FileCache fileCache; diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInputTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInputTests.java index 7fb7a03584e20..bc646cc8d50db 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInputTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FullFileCachedIndexInputTests.java @@ -8,17 +8,22 @@ package org.opensearch.index.store.remote.filecache; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.store.IndexInput; +import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter; import java.io.IOException; +import java.util.concurrent.TimeUnit; +@ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class) public class FullFileCachedIndexInputTests extends FileCachedIndexInputTests { private FullFileCachedIndexInput fullFileCachedIndexInput; @Override protected void setupIndexInputAndAddToFileCache() { fullFileCachedIndexInput = new FullFileCachedIndexInput(fileCache, filePath, underlyingIndexInput); + // Putting in the file cache would increase refCount to 1 fileCache.put(filePath, new CachedFullFileIndexInput(fileCache, filePath, fullFileCachedIndexInput)); } @@ -37,15 +42,11 @@ public void testClone() throws IOException { fileCache.decRef(filePath); assertFalse(isActiveAndTotalUsageSame()); - // After cloning the refCount will increase again and activeUsage and totalUsage will be same again - FileCachedIndexInput clonedFileCachedIndexInput1 = fullFileCachedIndexInput.clone(); - FileCachedIndexInput clonedFileCachedIndexInput2 = clonedFileCachedIndexInput1.clone(); - FileCachedIndexInput clonedFileCachedIndexInput3 = clonedFileCachedIndexInput2.clone(); - assertTrue(isActiveAndTotalUsageSame()); + // Since no clones have been done, refCount should be zero + assertEquals((int) fileCache.getRef(filePath), 0); - // closing the first level clone will close all subsequent level clones and reduce ref count to 0 - clonedFileCachedIndexInput1.close(); - assertFalse(isActiveAndTotalUsageSame()); + createUnclosedClonesSlices(false); + triggerGarbageCollectionAndAssertClonesClosed(); fileCache.prune(); @@ -68,12 +69,38 @@ public void testSlice() throws IOException { fileCache.decRef(filePath); assertFalse(isActiveAndTotalUsageSame()); - // Creating a slice will increase the refCount - IndexInput slicedFileCachedIndexInput = fullFileCachedIndexInput.slice(SLICE_DESC, 1, 2); - assertTrue(isActiveAndTotalUsageSame()); + // Since no clones have been done, refCount should be zero + assertEquals((int) fileCache.getRef(filePath), 0); + + createUnclosedClonesSlices(true); + triggerGarbageCollectionAndAssertClonesClosed(); - // Closing the parent will close all the slices as well decreasing the refCount to 0 - fullFileCachedIndexInput.close(); assertFalse(isActiveAndTotalUsageSame()); } + + private void triggerGarbageCollectionAndAssertClonesClosed() { + try { + // Clones/Slices will be phantom reachable now, triggering gc should call close on them + assertBusy(() -> { + System.gc(); // Do not rely on GC to be deterministic, hence the polling + assertEquals( + "Expected refCount to drop to zero as all clones/slices should have closed", + (int) fileCache.getRef(filePath), + 0 + ); + }, 5, TimeUnit.SECONDS); + } catch (Exception e) { + logger.error("Exception thrown while triggering gc", e); + fail(); + } + } + + private void createUnclosedClonesSlices(boolean createSlice) throws IOException { + int NUM_OF_CLONES = 3; + for (int i = 0; i < NUM_OF_CLONES; i++) { + if (createSlice) fullFileCachedIndexInput.slice("slice", 1, 2); + else fullFileCachedIndexInput.clone(); + } + assertEquals((int) fileCache.getRef(filePath), NUM_OF_CLONES); + } } From a04c5baca484afaf63d6bed78aca49e5e2df7509 Mon Sep 17 00:00:00 2001 From: Gagan6164 <32809202+Gagan6164@users.noreply.github.com> Date: Mon, 7 Jul 2025 16:51:54 +0530 Subject: [PATCH 20/51] Disallow resize for Warm Index, add Parameterized ITs for close in remote store (#18686) Signed-off-by: Gagan Singh Saini Co-authored-by: Gaurav Bafna --- CHANGELOG.md | 1 + .../indices/state/CloseIndexIT.java | 22 +++++- .../indices/state/RemoteCloseIndexIT.java | 74 +++++++++++++++++++ .../remotestore/WritableWarmIT.java | 43 +++++++++++ .../indices/shrink/TransportResizeAction.java | 5 ++ 5 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/indices/state/RemoteCloseIndexIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 0babaf80b126b..d42b7bc5ed1a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Add support for Warm Indices Write Block on Flood Watermark breach ([#18375](https://github.com/opensearch-project/OpenSearch/pull/18375)) - Add support for custom index name resolver from cluster plugin ([#18593](https://github.com/opensearch-project/OpenSearch/pull/18593)) +- Disallow resize for Warm Index, add Parameterized ITs for close in remote store ([#18686](https://github.com/opensearch-project/OpenSearch/pull/18686)) - Ability to run Code Coverage with Gradle and produce the jacoco reports locally ([#18509](https://github.com/opensearch-project/OpenSearch/issues/18509)) - Add NodeResourceUsageStats to ClusterInfo ([#18480](https://github.com/opensearch-project/OpenSearch/issues/18472)) - Introduce SecureHttpTransportParameters experimental API (to complement SecureTransportParameters counterpart) ([#18572](https://github.com/opensearch-project/OpenSearch/issues/18572)) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java index 6ad69698c774e..77d63eb20002a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java @@ -32,6 +32,8 @@ package org.opensearch.indices.state; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.admin.indices.close.CloseIndexRequestBuilder; @@ -56,11 +58,13 @@ import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.test.BackgroundIndexer; import org.opensearch.test.InternalTestCluster; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; import org.opensearch.transport.client.Client; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.concurrent.CountDownLatch; @@ -70,6 +74,7 @@ import static java.util.Collections.emptySet; import static java.util.stream.Collectors.toList; import static org.opensearch.action.support.IndicesOptions.lenientExpandOpen; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.internal.SearchContext.TRACK_TOTAL_HITS_ACCURATE; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @@ -82,10 +87,23 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class CloseIndexIT extends OpenSearchIntegTestCase { +public class CloseIndexIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { private static final int MAX_DOCS = 25_000; + public CloseIndexIT(Settings nodeSettings) { + super(nodeSettings); + } + + // This is to reuse CloseIndexIT in RemoteCloseIndexIT . + // Concurrent search deosn't make a difference in these tests. + @ParametersFactory + public static Collection parameters() { + return Collections.singletonList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() } + ); + } + @Override public Settings indexSettings() { return Settings.builder() diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/state/RemoteCloseIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/state/RemoteCloseIndexIT.java new file mode 100644 index 0000000000000..3267da06a99bb --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/indices/state/RemoteCloseIndexIT.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices.state; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.index.IndexModule; +import org.opensearch.index.IndexSettings; +import org.opensearch.node.Node; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collection; + +import static org.opensearch.common.util.FeatureFlags.WRITABLE_WARM_INDEX_SETTING; + +public class RemoteCloseIndexIT extends CloseIndexIT { + + public RemoteCloseIndexIT(Settings nodeSettings) { + super(nodeSettings); + } + + protected Path remoteRepoPath; + + protected final static String TEST_REMOTE_STORE_REPO_SUFFIX = "__rs"; + protected static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + if (remoteRepoPath == null) { + remoteRepoPath = randomRepoPath().toAbsolutePath(); + } + ByteSizeValue cacheSize = new ByteSizeValue(16, ByteSizeUnit.GB); + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath)) + .put(Node.NODE_SEARCH_CACHE_SIZE_SETTING.getKey(), cacheSize.toString()) + .build(); + } + + protected Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) { + Settings.Builder settingsBuilder = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s"); + if (WRITABLE_WARM_INDEX_SETTING.get(settings)) { + settingsBuilder.put(IndexModule.IS_WARM_INDEX_SETTING.getKey(), true); + } + return settingsBuilder; + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(WRITABLE_WARM_INDEX_SETTING.getKey(), true).build() }, + new Object[] { Settings.builder().put(WRITABLE_WARM_INDEX_SETTING.getKey(), false).build() } + ); + } + + void assertNoFileBasedRecovery(String indexName) { + // skipping for remote store + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/WritableWarmIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/WritableWarmIT.java index 9ae3ae5b7a451..a94d04cf17a06 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/WritableWarmIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/WritableWarmIT.java @@ -18,6 +18,7 @@ import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.get.GetIndexRequest; import org.opensearch.action.admin.indices.get.GetIndexResponse; +import org.opensearch.action.admin.indices.shrink.ResizeType; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; @@ -34,9 +35,11 @@ import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.store.remote.utils.FileTypeUtils; import org.opensearch.indices.IndicesService; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.node.Node; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.Assert; import java.util.Arrays; import java.util.HashSet; @@ -45,8 +48,11 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; +import static org.hamcrest.core.StringContains.containsString; @ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class) @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, supportsDedicatedMasters = false) @@ -247,4 +253,41 @@ public void testFullFileAndFileCacheStats() throws ExecutionException, Interrupt private boolean isFileCacheEmpty(AggregateFileCacheStats stats) { return stats.getUsed().getBytes() == 0L && stats.getActive().getBytes() == 0L; } + + public void testNoResizeOnWarm() { + InternalTestCluster internalTestCluster = internalCluster(); + internalTestCluster.startClusterManagerOnlyNode(); + internalCluster().startDataAndWarmNodes(1).get(0); + Settings idxSettings = Settings.builder() + .put(super.indexSettings()) + .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .put(SETTING_NUMBER_OF_SHARDS, 2) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexModule.IS_WARM_INDEX_SETTING.getKey(), true) + .build(); + + createIndex(INDEX_NAME, idxSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(Settings.builder().put("index.blocks.write", true)).get(); + ResizeType type = randomFrom(ResizeType.CLONE, ResizeType.SHRINK, ResizeType.SPLIT); + try { + client().admin() + .indices() + .prepareResizeIndex(INDEX_NAME, "target") + .setResizeType(type) + .setSettings( + Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", 2) + .putNull("index.blocks.write") + .build() + ) + .get(); + fail(); + } catch (Exception e) { + Assert.assertThat(e.getMessage(), containsString("cannot resize warm index")); + } + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java index 41b6df062ab42..0927ec234928a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java @@ -53,6 +53,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.IndexModule; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.DocsStats; @@ -146,6 +147,10 @@ protected void clusterManagerOperation( final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getSourceIndex()); final String targetIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getTargetIndexRequest().index()); IndexMetadata indexMetadata = state.metadata().index(sourceIndex); + if (indexMetadata.getSettings().getAsBoolean(IndexModule.IS_WARM_INDEX_SETTING.getKey(), false) == true) { + throw new IllegalStateException("cannot resize warm index"); + } + ClusterSettings clusterSettings = clusterService.getClusterSettings(); if (resizeRequest.getResizeType().equals(ResizeType.SHRINK) && state.metadata().isSegmentReplicationEnabled(sourceIndex) From fbbbecf619ed55b99f7829cfd4c1b510d5eefc21 Mon Sep 17 00:00:00 2001 From: shrugarg-amzn Date: Mon, 7 Jul 2025 23:23:13 +0530 Subject: [PATCH 21/51] Added support for range version support in semver (#18557) * Added support for range version support in semver Signed-off-by: Shruti Garg * Added test coverage for missing lines Signed-off-by: Shruti Garg * Fixed for non reachable code line and tests Signed-off-by: Shruti Garg --------- Signed-off-by: Shruti Garg --- .../org/opensearch/semver/SemverRange.java | 53 ++++- .../org/opensearch/semver/expr/Range.java | 95 +++++++++ .../opensearch/semver/SemverRangeTests.java | 199 ++++++++++++++++++ .../opensearch/semver/expr/RangeTests.java | 103 +++++++++ 4 files changed, 446 insertions(+), 4 deletions(-) create mode 100644 libs/core/src/main/java/org/opensearch/semver/expr/Range.java create mode 100644 libs/core/src/test/java/org/opensearch/semver/expr/RangeTests.java diff --git a/libs/core/src/main/java/org/opensearch/semver/SemverRange.java b/libs/core/src/main/java/org/opensearch/semver/SemverRange.java index da8c06c07d8e5..94ec876178ea5 100644 --- a/libs/core/src/main/java/org/opensearch/semver/SemverRange.java +++ b/libs/core/src/main/java/org/opensearch/semver/SemverRange.java @@ -16,11 +16,15 @@ import org.opensearch.semver.expr.Caret; import org.opensearch.semver.expr.Equal; import org.opensearch.semver.expr.Expression; +import org.opensearch.semver.expr.Range; import org.opensearch.semver.expr.Tilde; import java.io.IOException; +import java.util.Locale; import java.util.Objects; import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static java.util.Arrays.stream; @@ -31,6 +35,7 @@ *
  • '=' Requires exact match with the range version. For example, =1.2.3 range would match only 1.2.3
    • *
  • '~' Allows for patch version variability starting from the range version. For example, ~1.2.3 range would match versions greater than or equal to 1.2.3 but less than 1.3.0
    • *
  • '^' Allows for patch and minor version variability starting from the range version. For example, ^1.2.3 range would match versions greater than or equal to 1.2.3 but less than 2.0.0
    • + *
  • Explicit ranges: [2.0.0,3.0.0], (2.0.0,3.0.0), [2.0.0,3.0.0), (2.0.0,3.0.0]
    • * * * @opensearch.api @@ -38,12 +43,16 @@ @PublicApi(since = "2.13.0") public class SemverRange implements ToXContentFragment { + private static final Pattern RANGE_PATTERN = Pattern.compile("([\\[\\(])([\\d.]+)\\s*,\\s*([\\d.]+)([\\]\\)])"); + private final Version rangeVersion; private final RangeOperator rangeOperator; + private final Expression expression; public SemverRange(final Version rangeVersion, final RangeOperator rangeOperator) { this.rangeVersion = rangeVersion; this.rangeOperator = rangeOperator; + this.expression = rangeOperator.expression; } /** @@ -52,6 +61,23 @@ public SemverRange(final Version rangeVersion, final RangeOperator rangeOperator * @return a {@code SemverRange} */ public static SemverRange fromString(final String range) { + // Check if it's a range expression + Matcher matcher = RANGE_PATTERN.matcher(range); + if (matcher.matches()) { + char leftBracket = matcher.group(1).charAt(0); + String lowerVersionStr = matcher.group(2); + String upperVersionStr = matcher.group(3); + char rightBracket = matcher.group(4).charAt(0); + + Version lowerVersion = Version.fromString(matcher.group(2)); + Version upperVersion = Version.fromString(matcher.group(3)); + boolean includeLower = leftBracket == '['; + boolean includeUpper = rightBracket == ']'; + + Range rangeExpression = new Range(lowerVersion, upperVersion, includeLower, includeUpper); + return new SemverRange(lowerVersion, RangeOperator.RANGE, rangeExpression); + } + RangeOperator rangeOperator = RangeOperator.fromRange(range); String version = range.replaceFirst(rangeOperator.asEscapedString(), ""); if (!Version.stringHasLength(version)) { @@ -60,6 +86,12 @@ public static SemverRange fromString(final String range) { return new SemverRange(Version.fromString(version), rangeOperator); } + public SemverRange(Version rangeVersion, RangeOperator operator, Expression customExpression) { + this.rangeVersion = rangeVersion; + this.rangeOperator = operator; + this.expression = customExpression; + } + /** * Return the range operator for this range. * @return range operator @@ -94,7 +126,7 @@ public boolean isSatisfiedBy(final String versionToEvaluate) { * @see #isSatisfiedBy(String) */ public boolean isSatisfiedBy(final Version versionToEvaluate) { - return this.rangeOperator.expression.evaluate(this.rangeVersion, versionToEvaluate); + return this.expression.evaluate(this.rangeVersion, versionToEvaluate); } @Override @@ -106,16 +138,29 @@ public boolean equals(@Nullable final Object o) { return false; } SemverRange range = (SemverRange) o; - return Objects.equals(rangeVersion, range.rangeVersion) && rangeOperator == range.rangeOperator; + return Objects.equals(rangeVersion, range.rangeVersion) + && rangeOperator == range.rangeOperator + && Objects.equals(expression, range.expression); } @Override public int hashCode() { - return Objects.hash(rangeVersion, rangeOperator); + return Objects.hash(rangeVersion, rangeOperator, expression); } @Override public String toString() { + if (rangeOperator == RangeOperator.RANGE && expression instanceof Range) { + Range range = (Range) expression; + return String.format( + Locale.ROOT, + "%s%s,%s%s", + range.isIncludeLower() ? "[" : "(", + range.getLowerBound(), + range.getUpperBound(), + range.isIncludeUpper() ? "]" : ")" + ); + } return rangeOperator.asString() + rangeVersion; } @@ -128,10 +173,10 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa * A range operator. */ public enum RangeOperator { - EQ("=", new Equal()), TILDE("~", new Tilde()), CARET("^", new Caret()), + RANGE("range", new Range()), DEFAULT("", new Equal()); private final String operator; diff --git a/libs/core/src/main/java/org/opensearch/semver/expr/Range.java b/libs/core/src/main/java/org/opensearch/semver/expr/Range.java new file mode 100644 index 0000000000000..5fcc352a8b2ad --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/semver/expr/Range.java @@ -0,0 +1,95 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.semver.expr; + +import org.opensearch.Version; + +import java.util.Objects; + +/** + * Expression to evaluate version compatibility within a specified range with configurable bounds. + */ +public class Range implements Expression { + private final Version lowerBound; + private final Version upperBound; + private final boolean includeLower; + private final boolean includeUpper; + + public Range() { + this.lowerBound = Version.fromString("0.0.0"); // Minimum version + this.upperBound = Version.fromString("999.999.999"); // Maximum version + this.includeLower = true; // Default to inclusive bounds + this.includeUpper = true; + } + + public Range(Version lowerBound, Version upperBound, boolean includeLower, boolean includeUpper) { + if (lowerBound == null) { + throw new IllegalArgumentException("Lower bound cannot be null"); + } + if (upperBound == null) { + throw new IllegalArgumentException("Upper bound cannot be null"); + } + if (lowerBound.after(upperBound)) { + throw new IllegalArgumentException("Lower bound must be less than or equal to upper bound"); + } + this.lowerBound = lowerBound; + this.upperBound = upperBound; + this.includeLower = includeLower; + this.includeUpper = includeUpper; + } + + public void updateRange(Range other) { + if (other == null) { + throw new IllegalArgumentException("Range cannot be null"); + } + } + + @Override + public boolean evaluate(final Version rangeVersion, final Version versionToEvaluate) { + + boolean satisfiesLower = includeLower ? versionToEvaluate.onOrAfter(lowerBound) : versionToEvaluate.after(lowerBound); + + boolean satisfiesUpper = includeUpper ? versionToEvaluate.onOrBefore(upperBound) : versionToEvaluate.before(upperBound); + + return satisfiesLower && satisfiesUpper; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Range range = (Range) o; + return includeLower == range.includeLower + && includeUpper == range.includeUpper + && Objects.equals(lowerBound, range.lowerBound) + && Objects.equals(upperBound, range.upperBound); + } + + @Override + public int hashCode() { + return Objects.hash(lowerBound, upperBound, includeLower, includeUpper); + } + + public boolean isIncludeLower() { + return includeLower; + } + + public boolean isIncludeUpper() { + return includeUpper; + } + + public Version getLowerBound() { + return lowerBound; + } + + public Version getUpperBound() { + return upperBound; + } + +} diff --git a/libs/core/src/test/java/org/opensearch/semver/SemverRangeTests.java b/libs/core/src/test/java/org/opensearch/semver/SemverRangeTests.java index af1d95b2561b7..ba624d5287980 100644 --- a/libs/core/src/test/java/org/opensearch/semver/SemverRangeTests.java +++ b/libs/core/src/test/java/org/opensearch/semver/SemverRangeTests.java @@ -101,5 +101,204 @@ public void testInvalidRanges() { assertTrue(ex.getMessage().contains("the version needs to contain major, minor, and revision, and optionally the build")); expectThrows(NumberFormatException.class, () -> SemverRange.fromString("$1.2.3")); + + assertThrows(IllegalArgumentException.class, () -> SemverRange.fromString("[2.3.0]")); + assertThrows(IllegalArgumentException.class, () -> SemverRange.fromString("[2.3.0,]")); + assertThrows(IllegalArgumentException.class, () -> SemverRange.fromString("[,2.7.0]")); + assertThrows(IllegalArgumentException.class, () -> SemverRange.fromString("2.3.0,2.7.0")); + assertThrows(IllegalArgumentException.class, () -> SemverRange.fromString("[2.7.0,2.3.0]")); + } + + public void testInclusiveRange() { + SemverRange range = SemverRange.fromString("[2.3.0,2.7.0]"); + + // Test lower bound + assertTrue("Should include lower bound", range.isSatisfiedBy("2.3.0")); + + // Test upper bound + assertTrue("Should include upper bound", range.isSatisfiedBy("2.7.0")); + + // Test middle values + assertTrue("Should include values in range", range.isSatisfiedBy("2.5.0")); + assertTrue("Should include patch versions", range.isSatisfiedBy("2.4.1")); + + // Test out of range values + assertFalse("Should exclude values below range", range.isSatisfiedBy("2.2.9")); + assertFalse("Should exclude values above range", range.isSatisfiedBy("2.7.1")); } + + public void testExclusiveRange() { + SemverRange range = SemverRange.fromString("(2.3.0,2.7.0)"); + + // Test bounds + assertFalse("Should exclude lower bound", range.isSatisfiedBy("2.3.0")); + assertFalse("Should exclude upper bound", range.isSatisfiedBy("2.7.0")); + + // Test middle values + assertTrue("Should include values in range", range.isSatisfiedBy("2.5.0")); + assertTrue("Should include values near lower bound", range.isSatisfiedBy("2.3.1")); + assertTrue("Should include values near upper bound", range.isSatisfiedBy("2.6.9")); + + // Test out of range values + assertFalse("Should exclude values below range", range.isSatisfiedBy("2.2.9")); + assertFalse("Should exclude values above range", range.isSatisfiedBy("2.7.1")); + } + + public void testMixedRanges() { + // Test inclusive lower bound, exclusive upper bound + SemverRange range1 = SemverRange.fromString("[2.3.0,2.7.0)"); + assertTrue("Should include lower bound", range1.isSatisfiedBy("2.3.0")); + assertFalse("Should exclude upper bound", range1.isSatisfiedBy("2.7.0")); + assertTrue("Should include values in range", range1.isSatisfiedBy("2.6.9")); + + // Test exclusive lower bound, inclusive upper bound + SemverRange range2 = SemverRange.fromString("(2.3.0,2.7.0]"); + assertFalse("Should exclude lower bound", range2.isSatisfiedBy("2.3.0")); + assertTrue("Should include upper bound", range2.isSatisfiedBy("2.7.0")); + assertTrue("Should include values in range", range2.isSatisfiedBy("2.3.1")); + } + + public void testInvalidRangeConstructions() { + // Test invalid range constructions + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> SemverRange.fromString("[2.0.0,1.0.0]") // Lower bound greater than upper bound + ); + assertTrue(ex.getMessage().contains("Lower bound must be less than or equal to upper bound")); + + // Test malformed ranges + assertThrows(IllegalArgumentException.class, () -> SemverRange.fromString("[2.0.0]")); + assertThrows(IllegalArgumentException.class, () -> SemverRange.fromString("[2.0.0,]")); + assertThrows(IllegalArgumentException.class, () -> SemverRange.fromString("[,2.0.0]")); + } + + public void testEqualsAndHashCode() { + // Test inclusive ranges [a,b] + SemverRange inclusiveRange1 = SemverRange.fromString("[1.0.0,2.0.0]"); + SemverRange inclusiveRange2 = SemverRange.fromString("[1.0.0,2.0.0]"); + + assertTrue("Same inclusive range should be equal to itself", inclusiveRange1.equals(inclusiveRange1)); + assertTrue("Identical inclusive ranges should be equal", inclusiveRange1.equals(inclusiveRange2)); + assertEquals("Equal inclusive ranges should have same hashcode", inclusiveRange1.hashCode(), inclusiveRange2.hashCode()); + + // Test exclusive ranges (a,b) + SemverRange exclusiveRange1 = SemverRange.fromString("(1.0.0,2.0.0)"); + SemverRange exclusiveRange2 = SemverRange.fromString("(1.0.0,2.0.0)"); + + assertTrue("Same exclusive range should be equal to itself", exclusiveRange1.equals(exclusiveRange1)); + assertTrue("Identical exclusive ranges should be equal", exclusiveRange1.equals(exclusiveRange2)); + assertEquals("Equal exclusive ranges should have same hashcode", exclusiveRange1.hashCode(), exclusiveRange2.hashCode()); + + // Test mixed ranges [a,b) and (a,b] + SemverRange leftInclusiveRange1 = SemverRange.fromString("[1.0.0,2.0.0)"); + SemverRange leftInclusiveRange2 = SemverRange.fromString("[1.0.0,2.0.0)"); + SemverRange rightInclusiveRange1 = SemverRange.fromString("(1.0.0,2.0.0]"); + SemverRange rightInclusiveRange2 = SemverRange.fromString("(1.0.0,2.0.0]"); + + assertTrue("Same left-inclusive ranges should be equal", leftInclusiveRange1.equals(leftInclusiveRange2)); + assertTrue("Same right-inclusive ranges should be equal", rightInclusiveRange1.equals(rightInclusiveRange2)); + assertEquals( + "Equal left-inclusive ranges should have same hashcode", + leftInclusiveRange1.hashCode(), + leftInclusiveRange2.hashCode() + ); + assertEquals( + "Equal right-inclusive ranges should have same hashcode", + rightInclusiveRange1.hashCode(), + rightInclusiveRange2.hashCode() + ); + + // Test inequality between different range types + assertFalse("Inclusive range should not equal exclusive range", inclusiveRange1.equals(exclusiveRange1)); + assertFalse("Inclusive range should not equal left-inclusive range", inclusiveRange1.equals(leftInclusiveRange1)); + assertFalse("Inclusive range should not equal right-inclusive range", inclusiveRange1.equals(rightInclusiveRange1)); + assertFalse("Exclusive range should not equal left-inclusive range", exclusiveRange1.equals(leftInclusiveRange1)); + assertFalse("Left-inclusive range should not equal right-inclusive range", leftInclusiveRange1.equals(rightInclusiveRange1)); + + // Test different version ranges + SemverRange differentRange = SemverRange.fromString("[1.0.0,3.0.0]"); + assertFalse("Ranges with different versions should not be equal", inclusiveRange1.equals(differentRange)); + + // Test null and different types + assertFalse("Range should not equal null", inclusiveRange1.equals(null)); + assertFalse("Range should not equal different type", inclusiveRange1.equals("not a range")); + } + + public void testRangeOperatorExpression() { + // Test that expressions are properly assigned for different operators + SemverRange eqRange = SemverRange.fromString("=2.0.0"); + SemverRange tildeRange = SemverRange.fromString("~2.0.0"); + SemverRange caretRange = SemverRange.fromString("^2.0.0"); + + assertEquals(SemverRange.RangeOperator.EQ, eqRange.getRangeOperator()); + assertEquals(SemverRange.RangeOperator.TILDE, tildeRange.getRangeOperator()); + assertEquals(SemverRange.RangeOperator.CARET, caretRange.getRangeOperator()); + } + + public void testRangeToStringConsistency() { + // Test that toString produces consistent output + String[] testRanges = { "[2.0.0,3.0.0]", "(2.0.0,3.0.0)", "[2.0.0,3.0.0)", "(2.0.0,3.0.0]", "^2.0.0", "~2.0.0", "=2.0.0" }; + + for (String rangeStr : testRanges) { + SemverRange range = SemverRange.fromString(rangeStr); + assertEquals("toString should match original string", rangeStr, range.toString()); + } + } + + public void testRangeBoundaryConditions() { + // Test edge cases for range boundaries + SemverRange range = SemverRange.fromString("[2.0.0,3.0.0]"); + + // Test exact boundaries + assertTrue("Should include lower bound", range.isSatisfiedBy("2.0.0")); + assertTrue("Should include upper bound", range.isSatisfiedBy("3.0.0")); + + // Test just outside boundaries + assertFalse("Should exclude version before lower bound", range.isSatisfiedBy("1.9.999")); + assertFalse("Should exclude version after upper bound", range.isSatisfiedBy("3.0.1")); + } + + public void testRangeEvaluationConsistency() { + // Test that range evaluation is consistent + SemverRange range = SemverRange.fromString("[2.0.0,3.0.0]"); + + // Test same version evaluation + boolean result1 = range.isSatisfiedBy("2.5.0"); + boolean result2 = range.isSatisfiedBy("2.5.0"); + assertEquals("Same version should evaluate consistently", result1, result2); + + } + + public void testRangeToString() { + // Test that toString produces the same string that was parsed + String[] ranges = { "[2.3.0,2.7.0]", "(2.3.0,2.7.0)", "[2.3.0,2.7.0)", "(2.3.0,2.7.0]" }; + + for (String rangeStr : ranges) { + SemverRange range = SemverRange.fromString(rangeStr); + assertEquals("toString should match original string", rangeStr, range.toString()); + } + } + + public void testRangeEquality() { + SemverRange range1 = SemverRange.fromString("[2.3.0,2.7.0]"); + SemverRange range2 = SemverRange.fromString("[2.3.0,2.7.0]"); + SemverRange range3 = SemverRange.fromString("(2.3.0,2.7.0]"); + + assertEquals("Identical ranges should be equal", range1, range2); + assertNotEquals("Different ranges should not be equal", range1, range3); + assertNotEquals("Range should not equal null", null, range1); + } + + public void testVersionEdgeCases() { + SemverRange range = SemverRange.fromString("[2.0.0,3.0.0]"); + + // Test major version boundaries + assertTrue(range.isSatisfiedBy("2.0.0")); + assertTrue(range.isSatisfiedBy("2.99.99")); + assertTrue(range.isSatisfiedBy("3.0.0")); + assertFalse(range.isSatisfiedBy("1.99.99")); + assertFalse(range.isSatisfiedBy("3.0.1")); + + } + } diff --git a/libs/core/src/test/java/org/opensearch/semver/expr/RangeTests.java b/libs/core/src/test/java/org/opensearch/semver/expr/RangeTests.java new file mode 100644 index 0000000000000..a9e953ef5b270 --- /dev/null +++ b/libs/core/src/test/java/org/opensearch/semver/expr/RangeTests.java @@ -0,0 +1,103 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.semver.expr; + +import org.opensearch.Version; +import org.opensearch.test.OpenSearchTestCase; + +public class RangeTests extends OpenSearchTestCase { + + public void testDefaultConstructor() { + Range range = new Range(); + assertEquals(Version.fromString("0.0.0"), range.getLowerBound()); + assertEquals(Version.fromString("999.999.999"), range.getUpperBound()); + assertTrue(range.isIncludeLower()); + assertTrue(range.isIncludeUpper()); + + } + + public void testRangeEvaluation() { + Version lowerBound = Version.fromString("2.3.0"); + Version upperBound = Version.fromString("2.7.0"); + + // Test inclusive range + Range inclusiveRange = new Range(lowerBound, upperBound, true, true); + assertTrue(inclusiveRange.evaluate(null, Version.fromString("2.3.0"))); + assertTrue(inclusiveRange.evaluate(null, Version.fromString("2.5.0"))); + assertTrue(inclusiveRange.evaluate(null, Version.fromString("2.7.0"))); + assertFalse(inclusiveRange.evaluate(null, Version.fromString("2.2.9"))); + assertFalse(inclusiveRange.evaluate(null, Version.fromString("2.7.1"))); + + // Test exclusive range + Range exclusiveRange = new Range(lowerBound, upperBound, false, false); + assertFalse(exclusiveRange.evaluate(null, Version.fromString("2.3.0"))); + assertTrue(exclusiveRange.evaluate(null, Version.fromString("2.5.0"))); + assertFalse(exclusiveRange.evaluate(null, Version.fromString("2.7.0"))); + assertFalse(exclusiveRange.evaluate(null, Version.fromString("2.2.9"))); + assertFalse(exclusiveRange.evaluate(null, Version.fromString("2.7.1"))); + } + + public void testInvalidRanges() { + Version lowerBound = Version.fromString("2.3.0"); + Version upperBound = Version.fromString("2.7.0"); + + // Test null bounds + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new Range(null, upperBound, true, true)); + assertEquals("Lower bound cannot be null", ex.getMessage()); + + ex = expectThrows(IllegalArgumentException.class, () -> new Range(lowerBound, null, true, true)); + assertEquals("Upper bound cannot be null", ex.getMessage()); + + // Test invalid range (upper < lower) + ex = expectThrows(IllegalArgumentException.class, () -> new Range(upperBound, lowerBound, true, true)); + assertEquals("Lower bound must be less than or equal to upper bound", ex.getMessage()); + } + + public void testMixedBoundTypes() { + Version lowerBound = Version.fromString("2.3.0"); + Version upperBound = Version.fromString("2.7.0"); + + // Test inclusive lower, exclusive upper + Range mixedRange1 = new Range(lowerBound, upperBound, true, false); + assertTrue(mixedRange1.evaluate(null, Version.fromString("2.3.0"))); + assertTrue(mixedRange1.evaluate(null, Version.fromString("2.6.9"))); + assertFalse(mixedRange1.evaluate(null, Version.fromString("2.7.0"))); + + // Test exclusive lower, inclusive upper + Range mixedRange2 = new Range(lowerBound, upperBound, false, true); + assertFalse(mixedRange2.evaluate(null, Version.fromString("2.3.0"))); + assertTrue(mixedRange2.evaluate(null, Version.fromString("2.3.1"))); + assertTrue(mixedRange2.evaluate(null, Version.fromString("2.7.0"))); + } + + public void testRangeAccessors() { + Version lowerBound = Version.fromString("2.3.0"); + Version upperBound = Version.fromString("2.7.0"); + Range range = new Range(lowerBound, upperBound, true, false); + + assertEquals("Lower bound should match", lowerBound, range.getLowerBound()); + assertEquals("Upper bound should match", upperBound, range.getUpperBound()); + assertTrue("Should be inclusive lower", range.isIncludeLower()); + assertFalse("Should be exclusive upper", range.isIncludeUpper()); + } + + public void testUpdateRangeNullCheck() { + Range range = new Range( + Version.fromString("1.0.0"), + Version.fromString("2.0.0"), + true, // includeLower + true // includeUpper + ); + + // Test that updateRange throws IllegalArgumentException for null input + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> range.updateRange(null)); + assertEquals("Range cannot be null", ex.getMessage()); + } + +} From 504d956515d6cf789e25f745bed293b1032416a6 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 7 Jul 2025 14:23:12 -0400 Subject: [PATCH 22/51] An attempt to fix flaky SecureReactorNetty4HttpServerTransportTests (#18680) Signed-off-by: Andriy Redko --- .../reactor/netty4/ReactorNetty4HttpServerTransport.java | 1 + .../opensearch/http/reactor/netty4/ReactorHttpClient.java | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java index da0aa229e6059..6ddc98ba9d22e 100644 --- a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java +++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java @@ -243,6 +243,7 @@ protected HttpServerChannel bind(InetSocketAddress socketAddress) throws Excepti .runOn(sharedGroup.getLowLevelGroup()) .bindAddress(() -> socketAddress) .compress(true) + .http2Settings(spec -> spec.maxHeaderListSize(maxHeaderSize.bytesAsInt())) .httpRequestDecoder( spec -> spec.maxChunkSize(maxChunkSize.bytesAsInt()) .h2cMaxContentLength(h2cMaxContentLength.bytesAsInt()) diff --git a/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java index 7bdaef3ac2304..edd5bc97368dd 100644 --- a/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java +++ b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java @@ -25,6 +25,7 @@ import java.io.UncheckedIOException; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -56,6 +57,8 @@ import reactor.netty.http.Http2SslContextSpec; import reactor.netty.http.HttpProtocol; import reactor.netty.http.client.HttpClient; +import reactor.netty.http.client.PrematureCloseException; +import reactor.util.retry.Retry; import static io.netty.handler.codec.http.HttpHeaderNames.HOST; import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; @@ -255,6 +258,7 @@ private FullHttpResponse sendRequestStream( ) ) ) + .retryWhen(Retry.backoff(3, Duration.ofSeconds(1)).filter(throwable -> throwable instanceof PrematureCloseException)) .blockLast(); } finally { @@ -283,7 +287,7 @@ private HttpClient createClient(final InetSocketAddress remoteAddress, final Nio .configure(s -> s.clientAuth(ClientAuth.NONE).trustManager(InsecureTrustManagerFactory.INSTANCE)) : Http2SslContextSpec.forClient() .configure(s -> s.clientAuth(ClientAuth.NONE).trustManager(InsecureTrustManagerFactory.INSTANCE)) - ) + ).handshakeTimeout(Duration.ofSeconds(30)) ); } else { return client.protocol( From 871fe2957068cc2a84c95b2a6ae22c4cbb0c84e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 16:45:25 -0400 Subject: [PATCH 23/51] Bump com.squareup.okio:okio from 3.14.0 to 3.15.0 in /test/fixtures/hdfs-fixture (#18689) * Bump com.squareup.okio:okio in /test/fixtures/hdfs-fixture Bumps [com.squareup.okio:okio](https://github.com/square/okio) from 3.14.0 to 3.15.0. - [Release notes](https://github.com/square/okio/releases) - [Changelog](https://github.com/square/okio/blob/master/CHANGELOG.md) - [Commits](https://github.com/square/okio/compare/parent-3.14.0...parent-3.15.0) --- updated-dependencies: - dependency-name: com.squareup.okio:okio dependency-version: 3.15.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 +- test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d42b7bc5ed1a3..bf2f2dfb463ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.logging.log4j:log4j-core` from 2.24.3 to 2.25.0 ([#18589](https://github.com/opensearch-project/OpenSearch/pull/18589)) - Bump `com.google.code.gson:gson` from 2.13.0 to 2.13.1 ([#18585](https://github.com/opensearch-project/OpenSearch/pull/18585)) - Bump `com.azure:azure-core-http-netty` from 1.15.11 to 1.15.12 ([#18586](https://github.com/opensearch-project/OpenSearch/pull/18586)) -- Bump `com.squareup.okio:okio` from 3.13.0 to 3.14.0 ([#18645](https://github.com/opensearch-project/OpenSearch/pull/18645)) +- Bump `com.squareup.okio:okio` from 3.13.0 to 3.15.0 ([#18645](https://github.com/opensearch-project/OpenSearch/pull/18645), [#18689](https://github.com/opensearch-project/OpenSearch/pull/18689)) - Bump `com.netflix.nebula.ospackage-base` from 11.11.2 to 12.0.0 ([#18646](https://github.com/opensearch-project/OpenSearch/pull/18646)) - Bump `com.azure:azure-storage-blob` from 12.30.0 to 12.30.1 ([#18644](https://github.com/opensearch-project/OpenSearch/pull/18644)) - Bump `org.bouncycastle:bctls-fips` from 2.0.19 to 2.0.20 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 6d6851e2eab50..9bbb8f86856ce 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -88,6 +88,6 @@ dependencies { runtimeOnly("com.squareup.okhttp3:okhttp:4.12.0") { exclude group: "com.squareup.okio" } - runtimeOnly "com.squareup.okio:okio:3.14.0" + runtimeOnly "com.squareup.okio:okio:3.15.0" runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.7" } From f4ca647be8430903a3343bcec48ced6794faa819 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Mon, 7 Jul 2025 13:52:03 -0700 Subject: [PATCH 24/51] Add Semantic Version field type mapper (#18454) * Add Semantic Version field type mapper and extensive unit tests Signed-off-by: Siddhant Deshmukh * Refactor SemanticVersionFieldMapper for flexible index, store, and doc_values support Signed-off-by: Siddhant Deshmukh --------- Signed-off-by: Siddhant Deshmukh --- CHANGELOG.md | 1 + .../index/mapper/SemanticVersion.java | 257 ++++++ .../mapper/SemanticVersionFieldMapper.java | 395 ++++++++ .../org/opensearch/indices/IndicesModule.java | 2 + .../SemanticVersionFieldMapperTests.java | 854 ++++++++++++++++++ .../index/mapper/SemanticVersionTests.java | 352 ++++++++ .../query/SemanticVersionFieldQueryTests.java | 419 +++++++++ .../aggregations/AggregatorTestCase.java | 2 + 8 files changed, 2282 insertions(+) create mode 100644 server/src/main/java/org/opensearch/index/mapper/SemanticVersion.java create mode 100644 server/src/main/java/org/opensearch/index/mapper/SemanticVersionFieldMapper.java create mode 100644 server/src/test/java/org/opensearch/index/mapper/SemanticVersionFieldMapperTests.java create mode 100644 server/src/test/java/org/opensearch/index/mapper/SemanticVersionTests.java create mode 100644 server/src/test/java/org/opensearch/index/query/SemanticVersionFieldQueryTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index bf2f2dfb463ca..15d826dd6c20a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add functionality for plugins to inject QueryCollectorContext during QueryPhase ([#18637](https://github.com/opensearch-project/OpenSearch/pull/18637)) - Add support for non-timing info in profiler ([#18460](https://github.com/opensearch-project/OpenSearch/issues/18460)) - Extend Approximation Framework to other numeric types ([#18530](https://github.com/opensearch-project/OpenSearch/issues/18530)) +- Add Semantic Version field type mapper and extensive unit tests([#18454](https://github.com/opensearch-project/OpenSearch/pull/18454)) ### Changed - Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570)) diff --git a/server/src/main/java/org/opensearch/index/mapper/SemanticVersion.java b/server/src/main/java/org/opensearch/index/mapper/SemanticVersion.java new file mode 100644 index 0000000000000..812d65c5494e8 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/SemanticVersion.java @@ -0,0 +1,257 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import java.util.Arrays; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Represents a semantic version number (major.minor.patch-preRelease+build). + * This class implements semantic versioning (SemVer) according to the specification at semver.org. + * It provides methods to parse, compare, and manipulate semantic version numbers. + * Primarily used in {@link SemanticVersionFieldMapper} for mapping and sorting purposes. + * + * @see Semantic Versioning 2.0.0 + * @see OpenSearch github issue + */ +public class SemanticVersion implements Comparable { + + // Regex used to check SemVer string. Source: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + private static final String SEMANTIC_VERSION_REGEX = + "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"; + private final int major; + private final int minor; + private final int patch; + private final String preRelease; + private final String build; + + public SemanticVersion(int major, int minor, int patch, String preRelease, String build) { + if (major < 0 || minor < 0 || patch < 0) { + throw new IllegalArgumentException("Version numbers cannot be negative"); + } + this.major = major; + this.minor = minor; + this.patch = patch; + this.preRelease = preRelease; + this.build = build; + } + + public int getMajor() { + return major; + } + + public int getMinor() { + return minor; + } + + public int getPatch() { + return patch; + } + + public String getPreRelease() { + return preRelease; + } + + public String getBuild() { + return build; + } + + public static SemanticVersion parse(String version) { + if (version == null || version.isEmpty()) { + throw new IllegalArgumentException("Version string cannot be null or empty"); + } + + // Clean up the input string + version = version.trim(); + version = version.replaceAll("\\[|\\]", ""); // Remove square brackets + + // Handle encoded byte format + if (version.matches(".*\\s+.*")) { + version = version.replaceAll("\\s+", "."); + } + + Pattern pattern = Pattern.compile(SEMANTIC_VERSION_REGEX); + + Matcher matcher = pattern.matcher(version); + if (!matcher.matches()) { + throw new IllegalArgumentException("Invalid semantic version format: [" + version + "]"); + } + + try { + return new SemanticVersion( + Integer.parseInt(matcher.group(1)), + Integer.parseInt(matcher.group(2)), + Integer.parseInt(matcher.group(3)), + matcher.group(4), + matcher.group(5) + ); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid version numbers in: " + version, e); + } + } + + /** + * Returns a normalized string representation of the semantic version. + * This format ensures proper lexicographical ordering of versions. + * The format is: + * - Major, minor, and patch numbers are padded to 20 digits + * - Pre-release version is appended with a "-" prefix if present + * - Build metadata is appended with a "+" prefix if present + * + * Example: "1.2.3-alpha+build.123" becomes "00000000000000000001.00000000000000000002.00000000000000000003-alpha+build.123" + * + * Note: Build metadata is included for completeness but does not affect version precedence + * as per SemVer specification. + * + * @return normalized string representation of the version + */ + public String getNormalizedString() { + StringBuilder sb = new StringBuilder(); + + // Pad numbers to 20 digits for consistent lexicographical sorting + // This allows for very large version numbers while maintaining proper order + sb.append(padWithZeros(major, 20)).append('.').append(padWithZeros(minor, 20)).append('.').append(padWithZeros(patch, 20)); + + // Add pre-release version if present + // Pre-release versions have lower precedence than the associated normal version + if (preRelease != null) { + sb.append('-').append(preRelease); + } + + // Add build metadata if present + // Note: Build metadata does not affect version precedence + if (build != null) { + sb.append('+').append(build); + } + + return sb.toString(); + } + + /** + * Returns a normalized comparable string representation of the semantic version. + *

    + * The format zero-pads major, minor, and patch versions to 20 digits each, + * separated by dots, to ensure correct lexical sorting of numeric components. + *

    + * For pre-release versions, the pre-release label is appended with a leading + * hyphen (`-`) in lowercase, preserving lexicographical order among pre-release + * versions. + *

    + * For stable releases (no pre-release), a tilde character (`~`) is appended, + * which lexically sorts after any pre-release versions to ensure stable + * releases are ordered last. + *

    + * Ordering: 1.0.0-alpha < 1.0.0-beta < 1.0.0 + *

    + * Examples: + *

      + *
    • 1.0.0 → 00000000000000000001.00000000000000000000.00000000000000000000~
      • + *
    • 1.0.0-alpha → 00000000000000000001.00000000000000000000.00000000000000000000-alpha
      • + *
    • 1.0.0-beta → 00000000000000000001.00000000000000000000.00000000000000000000-beta
      • + *
    + * + * @return normalized string for lexicographical comparison of semantic versions + */ + public String getNormalizedComparableString() { + StringBuilder sb = new StringBuilder(); + + // Zero-pad major, minor, patch + sb.append(padWithZeros(major, 20)).append("."); + sb.append(padWithZeros(minor, 20)).append("."); + sb.append(padWithZeros(patch, 20)); + + if (preRelease == null || preRelease.isEmpty()) { + // Stable release: append '~' to sort AFTER any pre-release + sb.append("~"); + } else { + // Pre-release: append '-' plus normalized pre-release string (lowercase, trimmed) + sb.append("-").append(preRelease.trim().toLowerCase(Locale.ROOT)); + } + + return sb.toString(); + } + + @Override + public int compareTo(SemanticVersion other) { + if (other == null) { + return 1; + } + + int majorComparison = Integer.compare(this.major, other.major); + if (majorComparison != 0) return majorComparison; + + int minorComparison = Integer.compare(this.minor, other.minor); + if (minorComparison != 0) return minorComparison; + + int patchComparison = Integer.compare(this.patch, other.patch); + if (patchComparison != 0) return patchComparison; + + // Pre-release versions have lower precedence + if (this.preRelease == null && other.preRelease != null) return 1; + if (this.preRelease != null && other.preRelease == null) return -1; + if (this.preRelease != null && other.preRelease != null) { + return comparePreRelease(this.preRelease, other.preRelease); + } + + return 0; + } + + private int comparePreRelease(String pre1, String pre2) { + String[] parts1 = pre1.split("\\."); + String[] parts2 = pre2.split("\\."); + + int length = Math.min(parts1.length, parts2.length); + for (int i = 0; i < length; i++) { + String part1 = parts1[i]; + String part2 = parts2[i]; + + boolean isNum1 = part1.matches("\\d+"); + boolean isNum2 = part2.matches("\\d+"); + + if (isNum1 && isNum2) { + int num1 = Integer.parseInt(part1); + int num2 = Integer.parseInt(part2); + int comparison = Integer.compare(num1, num2); + if (comparison != 0) return comparison; + } else { + int comparison = part1.compareTo(part2); + if (comparison != 0) return comparison; + } + } + + return Integer.compare(parts1.length, parts2.length); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(major).append('.').append(minor).append('.').append(patch); + if (preRelease != null) { + sb.append('-').append(preRelease); + } + if (build != null) { + sb.append('+').append(build); + } + return sb.toString(); + } + + private static String padWithZeros(long value, int width) { + String str = Long.toString(value); + int padding = width - str.length(); + if (padding > 0) { + char[] zeros = new char[padding]; + Arrays.fill(zeros, '0'); + return new String(zeros) + str; + } + return str; + } + +} diff --git a/server/src/main/java/org/opensearch/index/mapper/SemanticVersionFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/SemanticVersionFieldMapper.java new file mode 100644 index 0000000000000..3e3222ff5df75 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/SemanticVersionFieldMapper.java @@ -0,0 +1,395 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.KeywordField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.RegexpQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Operations; +import org.opensearch.common.unit.Fuzziness; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.fielddata.IndexFieldData; +import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.QueryShardException; +import org.opensearch.search.aggregations.support.CoreValuesSourceType; +import org.opensearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.Supplier; + +/** + * A field mapper for a semantic version field + * + */ +public class SemanticVersionFieldMapper extends ParametrizedFieldMapper { + public static final String CONTENT_TYPE = "version"; + public static final FieldType FIELD_TYPE = new FieldType(); + public static final String NORMALIZED_FIELD_SUFFIX = "._normalized"; + + static { + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setStored(false); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.setDocValuesType(DocValuesType.SORTED_SET); + FIELD_TYPE.freeze(); + } + + private final Map meta; + + protected SemanticVersionFieldMapper( + String simpleName, + FieldType fieldType, + MappedFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + Map meta + ) { + super(simpleName, mappedFieldType, multiFields, copyTo); + this.meta = meta; + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + } + + /** + * Builder class for constructing the SemanticVersionFieldMapper. + */ + public static class Builder extends ParametrizedFieldMapper.Builder { + private final Parameter> meta = Parameter.metaParam(); + private final Parameter indexed = Parameter.indexParam(m -> toType(m).isSearchable, true).alwaysSerialize(); + private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); + private final Parameter stored = Parameter.storeParam(m -> toType(m).isStored, false); + + private static SemanticVersionFieldType toType(FieldMapper m) { + return (SemanticVersionFieldType) ((ParametrizedFieldMapper) m).mappedFieldType; + } + + public Builder(String name) { + super(name); + } + + @Override + protected List> getParameters() { + List> parameters = new ArrayList<>(); + parameters.add(indexed); + parameters.add(hasDocValues); + parameters.add(stored); + parameters.add(meta); + return parameters; + } + + @Override + public SemanticVersionFieldMapper build(BuilderContext context) { + FieldType fieldType = new FieldType(); + fieldType.setTokenized(false); + fieldType.setStored(stored.getValue()); + fieldType.setIndexOptions(indexed.getValue() ? IndexOptions.DOCS : IndexOptions.NONE); + fieldType.setDocValuesType(hasDocValues.getValue() ? DocValuesType.SORTED_SET : DocValuesType.NONE); + fieldType.freeze(); + return new SemanticVersionFieldMapper( + name, + fieldType, + new SemanticVersionFieldType( + buildFullName(context), + meta.getValue(), + indexed.getValue(), + hasDocValues.getValue(), + stored.getValue() + ), + multiFieldsBuilder.build(this, context), + copyTo.build(), + meta.getValue() + ); + } + } + + public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n)); + + /** + * The specific field type for SemanticVersionFieldMapper + * + * @opensearch.internal + */ + public static class SemanticVersionFieldType extends TermBasedFieldType { + private final Map meta; + private final String normalizedFieldName; + private final boolean isSearchable; + private final boolean hasDocValues; + private final boolean isStored; + + public SemanticVersionFieldType( + String name, + Map meta, + boolean isSearchable, + boolean hasDocValues, + boolean isStored + ) { + super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + this.meta = meta; + this.normalizedFieldName = name + NORMALIZED_FIELD_SUFFIX; + this.isSearchable = isSearchable; + this.hasDocValues = hasDocValues; + this.isStored = isStored; + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + if (value == null) { + throw new IllegalArgumentException("Cannot search for null value"); + } + BytesRef bytes = value instanceof BytesRef ? (BytesRef) value : new BytesRef(value.toString()); + Query indexQuery = isSearchable ? new TermQuery(new Term(name(), bytes)) : null; + Query dvQuery = hasDocValues ? SortedSetDocValuesField.newSlowExactQuery(normalizedFieldName, bytes) : null; + if (indexQuery != null && dvQuery != null) { + return new IndexOrDocValuesQuery(indexQuery, dvQuery); + } else if (indexQuery != null) { + return indexQuery; + } else if (dvQuery != null) { + return dvQuery; + } else { + throw new IllegalArgumentException("Field [" + name() + "] is neither indexed nor has doc_values enabled"); + } + } + + @Override + public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { + try { + BytesRef lower = null; + BytesRef upper = null; + if (lowerTerm != null) { + String lowerStr = (lowerTerm instanceof BytesRef) ? ((BytesRef) lowerTerm).utf8ToString() : lowerTerm.toString(); + SemanticVersion lowerVersion = SemanticVersion.parse(lowerStr); + lower = new BytesRef(lowerVersion.getNormalizedComparableString()); + } + if (upperTerm != null) { + String upperStr = (upperTerm instanceof BytesRef) ? ((BytesRef) upperTerm).utf8ToString() : upperTerm.toString(); + SemanticVersion upperVersion = SemanticVersion.parse(upperStr); + upper = new BytesRef(upperVersion.getNormalizedComparableString()); + } + Query indexQuery = isSearchable ? new TermRangeQuery(normalizedFieldName, lower, upper, includeLower, includeUpper) : null; + Query dvQuery = hasDocValues + ? SortedSetDocValuesField.newSlowRangeQuery(normalizedFieldName, lower, upper, includeLower, includeUpper) + : null; + if (indexQuery != null && dvQuery != null) { + return new IndexOrDocValuesQuery(indexQuery, dvQuery); + } else if (indexQuery != null) { + return indexQuery; + } else if (dvQuery != null) { + return dvQuery; + } else { + throw new IllegalArgumentException("Field [" + name() + "] is neither indexed nor has doc_values enabled"); + } + } catch (Exception e) { + throw new QueryShardException( + context, + "Failed to create range query for field [" + + name() + + "]. Lower term: [" + + (lowerTerm != null ? lowerTerm.toString() : "null") + + "], Upper term: [" + + (upperTerm != null ? upperTerm.toString() : "null") + + "]" + ); + } + } + + @Override + public Query termsQuery(List values, QueryShardContext context) { + List bytesList = new ArrayList<>(); + for (Object value : values) { + bytesList.add(value instanceof BytesRef ? (BytesRef) value : new BytesRef(value.toString())); + } + Query indexQuery = isSearchable ? new org.apache.lucene.search.TermInSetQuery(name(), bytesList) : null; + Query dvQuery = hasDocValues ? SortedSetDocValuesField.newSlowSetQuery(normalizedFieldName, bytesList) : null; + if (indexQuery != null && dvQuery != null) { + return new IndexOrDocValuesQuery(indexQuery, dvQuery); + } else if (indexQuery != null) { + return indexQuery; + } else if (dvQuery != null) { + return dvQuery; + } else { + throw new IllegalArgumentException("Field [" + name() + "] is neither indexed nor has doc_values enabled"); + } + } + + @Override + public Query regexpQuery( + String value, + int syntaxFlags, + int matchFlags, + int maxDeterminizedStates, + MultiTermQuery.RewriteMethod method, + QueryShardContext context + ) { + if (method == null) { + method = MultiTermQuery.CONSTANT_SCORE_REWRITE; + } + if (isSearchable) { + return new RegexpQuery( + new Term(name(), indexedValueForSearch(value)), + syntaxFlags, + matchFlags, + RegexpQuery.DEFAULT_PROVIDER, + maxDeterminizedStates, + method + ); + } else { + throw new IllegalArgumentException("Regexp queries require the field to be indexed"); + } + } + + @Override + public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) { + if (caseInsensitive) { + value = value.toLowerCase(Locale.ROOT); + } + if (isSearchable) { + return new WildcardQuery(new Term(name(), indexedValueForSearch(value)), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); + } else { + throw new IllegalArgumentException("Wildcard queries require the field to be indexed"); + } + } + + @Override + public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) { + if (method == null) { + method = MultiTermQuery.CONSTANT_SCORE_REWRITE; + } + if (isSearchable) { + return new PrefixQuery(new Term(name(), indexedValueForSearch(value)), method); + } else { + throw new IllegalArgumentException("Prefix queries require the field to be indexed"); + } + } + + @Override + public Query fuzzyQuery( + Object value, + Fuzziness fuzziness, + int prefixLength, + int maxExpansions, + boolean transpositions, + MultiTermQuery.RewriteMethod method, + QueryShardContext context + ) { + if (method == null) { + method = MultiTermQuery.CONSTANT_SCORE_REWRITE; + } + if (isSearchable) { + return new FuzzyQuery( + new Term(name(), indexedValueForSearch(value)), + fuzziness.asDistance(), + prefixLength, + maxExpansions, + transpositions, + method + ); + } else { + throw new IllegalArgumentException("Fuzzy queries require the field to be indexed"); + } + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new SourceValueFetcher(name(), context, format) { + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + }; + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + if (!hasDocValues) { + throw new IllegalArgumentException("Field [" + name() + "] does not have doc_values enabled"); + } + return new SortedSetOrdinalsIndexFieldData.Builder(normalizedFieldName, CoreValuesSourceType.BYTES); + } + + @Override + public Map meta() { + return meta; + } + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + String value = context.parser().textOrNull(); + if (value == null) { + return; + } + + SemanticVersion version = SemanticVersion.parse(value); + String versionString = version.toString(); + String normalizedValue = version.getNormalizedComparableString(); + BytesRef bytes = new BytesRef(versionString); + BytesRef normalizedValueBytes = new BytesRef(normalizedValue); + + // For retrieval: store original version string + if (fieldType().isStored()) { + context.doc().add(new StoredField(fieldType().name(), versionString)); + } + + // For searching (term queries): use original version string + if (fieldType().isSearchable()) { + context.doc().add(new KeywordField(fieldType().name(), bytes, this.fieldType.stored() ? Field.Store.YES : Field.Store.NO)); + } + + // For range queries and sorting: use normalized form + if (fieldType().hasDocValues() || fieldType().isSearchable()) { + context.doc().add(new KeywordField(fieldType().name() + NORMALIZED_FIELD_SUFFIX, normalizedValueBytes, Field.Store.NO)); + } + if (fieldType().hasDocValues()) { + context.doc().add(new SortedSetDocValuesField(fieldType().name() + NORMALIZED_FIELD_SUFFIX, normalizedValueBytes)); + } + } + + @Override + public ParametrizedFieldMapper.Builder getMergeBuilder() { + Builder builder = new Builder(name()); + builder.init(this); + return builder; + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } +} diff --git a/server/src/main/java/org/opensearch/indices/IndicesModule.java b/server/src/main/java/org/opensearch/indices/IndicesModule.java index ad2eb834b721e..a0386eaf57c24 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesModule.java +++ b/server/src/main/java/org/opensearch/indices/IndicesModule.java @@ -68,6 +68,7 @@ import org.opensearch.index.mapper.ObjectMapper; import org.opensearch.index.mapper.RangeType; import org.opensearch.index.mapper.RoutingFieldMapper; +import org.opensearch.index.mapper.SemanticVersionFieldMapper; import org.opensearch.index.mapper.SeqNoFieldMapper; import org.opensearch.index.mapper.SourceFieldMapper; import org.opensearch.index.mapper.StarTreeMapper; @@ -177,6 +178,7 @@ public static Map getMappers(List mappe mappers.put(DerivedFieldMapper.CONTENT_TYPE, DerivedFieldMapper.PARSER); mappers.put(WildcardFieldMapper.CONTENT_TYPE, WildcardFieldMapper.PARSER); mappers.put(StarTreeMapper.CONTENT_TYPE, new StarTreeMapper.TypeParser()); + mappers.put(SemanticVersionFieldMapper.CONTENT_TYPE, SemanticVersionFieldMapper.PARSER); for (MapperPlugin mapperPlugin : mapperPlugins) { for (Map.Entry entry : mapperPlugin.getMappers().entrySet()) { diff --git a/server/src/test/java/org/opensearch/index/mapper/SemanticVersionFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/SemanticVersionFieldMapperTests.java new file mode 100644 index 0000000000000..35bc16c512bc6 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/mapper/SemanticVersionFieldMapperTests.java @@ -0,0 +1,854 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.document.KeywordField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.MultiReader; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.RegExp; +import org.opensearch.common.geo.ShapeRelation; +import org.opensearch.common.unit.Fuzziness; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.QueryShardException; +import org.opensearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.mockito.Mockito.mock; + +/** + * Field mapper for OpenSearch semantic version field type + */ +public class SemanticVersionFieldMapperTests extends MapperTestCase { + + @Override + protected void minimalMapping(XContentBuilder b) throws IOException { + b.field("type", "version"); + b.field("store", true); + b.field("index", true); + } + + @Override + protected void registerParameters(ParameterChecker checker) { + // No additional parameters to register for version field type + } + + @Override + protected void metaMapping(XContentBuilder b) throws IOException { + minimalMapping(b); + } + + @Override + protected void writeFieldValue(XContentBuilder builder) throws IOException { + builder.value("1.0.0"); + } + + public void testDefaults() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "1.0.0"))); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertThat(fields.length, greaterThanOrEqualTo(2)); + + boolean hasDocValues = false; + boolean hasStoredField = false; + for (IndexableField field : fields) { + if (field.fieldType().docValuesType() == DocValuesType.SORTED_SET) { + hasDocValues = true; + } + if (field.fieldType().stored()) { + hasStoredField = true; + } + } + + assertTrue("Field should have doc values", hasDocValues); + assertTrue("Field should be stored", hasStoredField); + } + + public void testValidVersionValues() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + + // Test regular version + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "1.2.3"))); + IndexableField field = doc.rootDoc().getField("field"); + assertThat(field.stringValue(), equalTo("1.2.3")); + + // Test version with pre-release + doc = mapper.parse(source(b -> b.field("field", "1.2.3-alpha"))); + field = doc.rootDoc().getField("field"); + assertThat(field.stringValue(), equalTo("1.2.3-alpha")); + + // Test version with build metadata + doc = mapper.parse(source(b -> b.field("field", "1.2.3+build.123"))); + field = doc.rootDoc().getField("field"); + assertThat(field.stringValue(), equalTo("1.2.3+build.123")); + + // Test version with both pre-release and build metadata + doc = mapper.parse(source(b -> b.field("field", "1.2.3-alpha+build.123"))); + field = doc.rootDoc().getField("field"); + assertThat(field.stringValue(), equalTo("1.2.3-alpha+build.123")); + } + + public void testStoredValues() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + + // Test storing different version formats + List versions = Arrays.asList( + "1.0.0", + "1.0.0-alpha", + "1.0.0-alpha.1", + "1.0.0+build.123", + "1.0.0-beta+build.123", + "999999999.999999999.999999999" + ); + + for (String version : versions) { + ParsedDocument doc = mapper.parse(source(b -> b.field("field", version))); + IndexableField storedField = doc.rootDoc().getField("field"); + assertNotNull("Stored field should exist for " + version, storedField); + assertEquals("Stored value should match input for " + version, version, storedField.stringValue()); + } + } + + public void testDocValuesSorting() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "1.0.0"))); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + boolean hasDocValues = false; + for (IndexableField field : fields) { + if (field.fieldType().docValuesType() == DocValuesType.SORTED_SET) { + hasDocValues = true; + assertEquals(DocValuesType.SORTED_SET, field.fieldType().docValuesType()); + } + } + assertTrue("Field should have sorted doc values", hasDocValues); + } + + public void testNullHandling() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + + // Test null value + ParsedDocument doc = mapper.parse(source(b -> b.nullField("field"))); + assertThat(doc.rootDoc().getFields("field"), arrayWithSize(0)); + + // Test missing field + doc = mapper.parse(source(b -> {})); + assertThat(doc.rootDoc().getFields("field"), arrayWithSize(0)); + } + + public void testMalformedVersions() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + + // Test various malformed versions + List malformedVersions = Arrays.asList( + "1", + "1.0", + "v1.0.0", + "1.0.0.0", + "-1.0.0", + "1.0.0-", + "1.0.0+", + "01.0.0", + "1.0.0@invalid" + ); + + for (String malformed : malformedVersions) { + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> mapper.parse(source(b -> b.field("field", malformed))) + ); + assertTrue(e.getCause().getMessage().contains("Invalid semantic version format")); + } + } + + public void testMetadataFields() throws Exception { + XContentBuilder mapping = fieldMapping(this::minimalMapping); + DocumentMapper mapper = createDocumentMapper(mapping); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "1.0.0"))); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + boolean hasDocValues = false; + for (IndexableField field : fields) { + if (field.fieldType().docValuesType() == DocValuesType.SORTED_SET) { + hasDocValues = true; + break; + } + } + assertTrue("Field should have doc values", hasDocValues); + } + + public void testToXContent() throws IOException { + XContentBuilder mapping = fieldMapping(this::minimalMapping); + DocumentMapper mapper = createDocumentMapper(mapping); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + mapper.mapping().toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + String mappingString = builder.toString(); + assertTrue(mappingString.contains("\"type\":\"version\"")); + } + + public void testMultipleVersionFields() throws Exception { + XContentBuilder mapping = mapping(b -> { + b.startObject("version1").field("type", "version").field("store", true).endObject(); + b.startObject("version2").field("type", "version").field("store", true).endObject(); + }); + + DocumentMapper mapper = createDocumentMapper(mapping); + ParsedDocument doc = mapper.parse(source(b -> { + b.field("version1", "1.0.0"); + b.field("version2", "2.0.0"); + })); + + assertNotNull(doc.rootDoc().getField("version1")); + assertNotNull(doc.rootDoc().getField("version2")); + assertNotEquals(doc.rootDoc().getField("version1").stringValue(), doc.rootDoc().getField("version2").stringValue()); + } + + public void testMajorVersionComparison() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + + ParsedDocument doc1 = mapper.parse(source(b -> b.field("field", "2.0.0"))); + ParsedDocument doc2 = mapper.parse(source(b -> b.field("field", "1.9.9"))); + + SemanticVersion v1 = SemanticVersion.parse(doc1.rootDoc().getField("field").stringValue()); + SemanticVersion v2 = SemanticVersion.parse(doc2.rootDoc().getField("field").stringValue()); + + assertTrue("Major version comparison failed", v1.compareTo(v2) > 0); + } + + public void testComplexPreReleaseComparison() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + + List versions = Arrays.asList( + "1.0.0-alpha.beta", + "1.0.0-alpha.1", + "1.0.0-alpha", + "1.0.0-alpha.beta.2", + "1.0.0-beta", + "1.0.0-alpha.12", + "1.0.0-beta.2", + "1.0.0" + ); + + List expected = Arrays.asList( + "1.0.0-alpha", + "1.0.0-alpha.1", + "1.0.0-alpha.12", + "1.0.0-alpha.beta", + "1.0.0-alpha.beta.2", + "1.0.0-beta", + "1.0.0-beta.2", + "1.0.0" + ); + + testVersionSorting(mapper, versions, expected); + } + + public void testBuildMetadataEquality() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + + ParsedDocument doc1 = mapper.parse(source(b -> b.field("field", "1.0.0+build.1"))); + ParsedDocument doc2 = mapper.parse(source(b -> b.field("field", "1.0.0+build.2"))); + + SemanticVersion v1 = SemanticVersion.parse(doc1.rootDoc().getField("field").stringValue()); + SemanticVersion v2 = SemanticVersion.parse(doc2.rootDoc().getField("field").stringValue()); + + assertEquals("Build metadata should not affect equality", 0, v1.compareTo(v2)); + } + + public void testMultipleFieldsInDocument() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + b.startObject("version1").field("type", "version").field("store", true).endObject(); + b.startObject("version2").field("type", "version").field("store", true).endObject(); + })); + + ParsedDocument doc = mapper.parse(source(b -> { + b.field("version1", "1.0.0"); + b.field("version2", "2.0.0"); + })); + + assertNotNull(doc.rootDoc().getField("version1")); + assertNotNull(doc.rootDoc().getField("version2")); + assertNotEquals(doc.rootDoc().getField("version1").stringValue(), doc.rootDoc().getField("version2").stringValue()); + } + + public void testExtremeVersionNumbers() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + + // Test very large version numbers + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "999999999.999999999.999999999"))); + assertNotNull(doc.rootDoc().getField("field")); + + // Test version with many pre-release parts + doc = mapper.parse(source(b -> b.field("field", "1.0.0-alpha.beta.gamma.delta.epsilon"))); + assertNotNull(doc.rootDoc().getField("field")); + + // Test version with many build metadata parts + doc = mapper.parse(source(b -> b.field("field", "1.0.0+build.123.456.789.abc.def"))); + assertNotNull(doc.rootDoc().getField("field")); + } + + public void testMoreInvalidVersions() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + + // Test empty string + expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("field", "")))); + + // Test only dots + expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("field", "...")))); + + // Test invalid characters + expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("field", "1.0.0@")))); + + // Test leading zeros + expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("field", "01.2.3")))); + } + + private void testVersionSorting(DocumentMapper mapper, List input, List expected) throws Exception { + List actual = input.stream().map(v -> { + try { + ParsedDocument doc = mapper.parse(source(b -> b.field("field", v))); + return doc.rootDoc().getField("field").stringValue(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).sorted((v1, v2) -> { + SemanticVersion sv1 = SemanticVersion.parse(v1); + SemanticVersion sv2 = SemanticVersion.parse(v2); + return sv1.compareTo(sv2); + }).collect(Collectors.toList()); + + assertThat(actual, contains(expected.toArray())); + } + + public void testInvalidVersionValues() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + + // Test invalid version format + MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("field", "1.2")))); + assertTrue(e.getCause().getMessage().contains("Invalid semantic version format")); + + // Test negative version numbers + e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("field", "-1.2.3")))); + assertTrue(e.getCause().getMessage().contains("Invalid semantic version format")); + + // Test invalid pre-release format + e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("field", "1.2.3-")))); + assertTrue(e.getCause().getMessage().contains("Invalid semantic version format")); + } + + public void testVersionSorting() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + + // Create a list of versions in random order + List versions = Arrays.asList( + "1.0.0", + "1.10.0", + "1.2.0", + "1.0.0-alpha", + "1.0.0-alpha.1", + "1.0.0-beta", + "1.0.0-beta.2", + "1.0.0-beta.11", + "1.0.0-rc.1", + "1.0.0+build.123" + ); + Collections.shuffle(versions, random()); + + // Store documents with versions + List docs = new ArrayList<>(); + for (String version : versions) { + ParsedDocument doc = mapper.parse(source(b -> b.field("field", version))); + docs.add(doc); + } + + // Extract and sort versions + List sortedVersions = docs.stream().map(doc -> { + for (IndexableField field : doc.rootDoc().getFields("field")) { + if (field.fieldType().stored()) { + return field.stringValue(); + } + } + return null; + }).filter(Objects::nonNull).sorted((v1, v2) -> { + SemanticVersion sv1 = SemanticVersion.parse(v1); + SemanticVersion sv2 = SemanticVersion.parse(v2); + return sv1.compareTo(sv2); + }).map(v -> { + // Normalize version by removing build metadata + SemanticVersion sv = SemanticVersion.parse(v); + return new SemanticVersion(sv.getMajor(), sv.getMinor(), sv.getPatch(), sv.getPreRelease(), null).toString(); + }).collect(Collectors.toList()); + + // Verify each version individually + assertEquals("Wrong number of versions", 10, sortedVersions.size()); + assertEquals("1.0.0-alpha", sortedVersions.get(0)); + assertEquals("1.0.0-alpha.1", sortedVersions.get(1)); + assertEquals("1.0.0-beta", sortedVersions.get(2)); + assertEquals("1.0.0-beta.2", sortedVersions.get(3)); + assertEquals("1.0.0-beta.11", sortedVersions.get(4)); + assertEquals("1.0.0-rc.1", sortedVersions.get(5)); + assertEquals("1.0.0", sortedVersions.get(6)); + assertEquals("1.0.0", sortedVersions.get(7)); // build.123 is stripped + assertEquals("1.2.0", sortedVersions.get(8)); + assertEquals("1.10.0", sortedVersions.get(9)); + } + + public void testDocValues() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "1.0.0"))); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + boolean hasDocValues = false; + for (IndexableField field : fields) { + if (field.fieldType().docValuesType() == DocValuesType.SORTED_SET) { + hasDocValues = true; + break; + } + } + assertTrue("Field should have doc values", hasDocValues); + } + + public void testTermQuery() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "1.0.0"))); + + IndexableField[] fields = doc.rootDoc().getFields("field"); + boolean hasTermField = false; + for (IndexableField field : fields) { + if (field instanceof KeywordField) { + hasTermField = true; + break; + } + } + assertTrue("Field should have keyword term field", hasTermField); + } + + public void testNullValue() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + ParsedDocument doc = mapper.parse(source(b -> b.nullField("field"))); + assertThat(doc.rootDoc().getFields("field"), arrayWithSize(0)); + } + + public void testRangeQueries() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + MappedFieldType fieldType = ((FieldMapper) mapper.mappers().getMapper("field")).fieldType(); + + // Test various range scenarios + Query rangeQuery1 = fieldType.rangeQuery("1.0.0", "2.0.0", true, true, ShapeRelation.INTERSECTS, null, null, null); + assertTrue( + rangeQuery1 instanceof TermRangeQuery + || (rangeQuery1 instanceof IndexOrDocValuesQuery + && ((IndexOrDocValuesQuery) rangeQuery1).getIndexQuery() instanceof TermRangeQuery) + ); + + Query rangeQuery2 = fieldType.rangeQuery("1.0.0-alpha", "1.0.0", true, true, ShapeRelation.INTERSECTS, null, null, null); + assertTrue( + rangeQuery2 instanceof TermRangeQuery + || (rangeQuery2 instanceof IndexOrDocValuesQuery + && ((IndexOrDocValuesQuery) rangeQuery2).getIndexQuery() instanceof TermRangeQuery) + ); + + // Test null bounds + Query rangeQuery3 = fieldType.rangeQuery(null, "2.0.0", true, true, ShapeRelation.INTERSECTS, null, null, null); + assertTrue( + rangeQuery3 instanceof TermRangeQuery + || (rangeQuery3 instanceof IndexOrDocValuesQuery + && ((IndexOrDocValuesQuery) rangeQuery3).getIndexQuery() instanceof TermRangeQuery) + ); + + Query rangeQuery4 = fieldType.rangeQuery("1.0.0", null, true, true, ShapeRelation.INTERSECTS, null, null, null); + assertTrue( + rangeQuery4 instanceof TermRangeQuery + || (rangeQuery4 instanceof IndexOrDocValuesQuery + && ((IndexOrDocValuesQuery) rangeQuery4).getIndexQuery() instanceof TermRangeQuery) + ); + + // Test actual document matching + ParsedDocument doc1 = mapper.parse(source(b -> b.field("field", "1.5.0"))); + ParsedDocument doc2 = mapper.parse(source(b -> b.field("field", "2.5.0"))); + + // Should match doc1 but not doc2 + Query rangeQuery = fieldType.rangeQuery("1.0.0", "2.0.0", true, true, ShapeRelation.INTERSECTS, null, null, null); + + // Create readers and searcher + Directory dir1 = new ByteBuffersDirectory(); + Directory dir2 = new ByteBuffersDirectory(); + + // Index first document + IndexWriter writer1 = new IndexWriter(dir1, new IndexWriterConfig()); + writer1.addDocument(doc1.rootDoc()); + writer1.close(); + + // Index second document + IndexWriter writer2 = new IndexWriter(dir2, new IndexWriterConfig()); + writer2.addDocument(doc2.rootDoc()); + writer2.close(); + + // Create readers + IndexReader reader1 = DirectoryReader.open(dir1); + IndexReader reader2 = DirectoryReader.open(dir2); + + // Create MultiReader with array of readers + IndexReader[] readers = new IndexReader[] { reader1, reader2 }; + MultiReader multiReader = new MultiReader(readers, true); + + IndexSearcher searcher = new IndexSearcher(multiReader); + TopDocs hits = searcher.search(rangeQuery, 10); + + // Clean up + multiReader.close(); + dir1.close(); + dir2.close(); + } + + private IndexSearcher setupSearcher(DocumentMapper mapper) throws IOException { + List docs = Arrays.asList( + mapper.parse(source(b -> b.field("field", "1.0.0"))), + mapper.parse(source(b -> b.field("field", "1.0.1"))), + mapper.parse(source(b -> b.field("field", "1.1.0"))), + mapper.parse(source(b -> b.field("field", "2.0.0"))), + mapper.parse(source(b -> b.field("field", "1.0.0-alpha"))), + mapper.parse(source(b -> b.field("field", "1.0.0-beta"))), + mapper.parse(source(b -> b.field("field", "1.0.0+build.123"))) + ); + + Directory dir = new ByteBuffersDirectory(); + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig()); + for (ParsedDocument doc : docs) { + writer.addDocument(doc.rootDoc()); + } + writer.close(); + + return new IndexSearcher(DirectoryReader.open(dir)); + } + + public void testPrefixQuery() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + MappedFieldType fieldType = ((FieldMapper) mapper.mappers().getMapper("field")).fieldType(); + IndexSearcher searcher = setupSearcher(mapper); + + Query prefixQuery = fieldType.prefixQuery( + "1.0", + MultiTermQuery.CONSTANT_SCORE_REWRITE, // Specify rewrite method + false, + null + ); + + assertThat("Should match 1.0.0, 1.0.1, 1.0.0-alpha, 1.0.0-beta, 1.0.0+build.123", searcher.count(prefixQuery), equalTo(5)); + + // Test different prefix patterns + Query majorVersionPrefix = fieldType.prefixQuery("1.", MultiTermQuery.CONSTANT_SCORE_REWRITE, false, null); + assertThat("Should match all version 1.x.x", searcher.count(majorVersionPrefix), equalTo(6)); + + // Test case sensitivity + Query caseInsensitivePrefix = fieldType.prefixQuery("1.0", MultiTermQuery.CONSTANT_SCORE_REWRITE, true, null); + assertThat("Should match same as case sensitive", searcher.count(caseInsensitivePrefix), equalTo(5)); + } + + public void testWildcardQuery() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + MappedFieldType fieldType = ((FieldMapper) mapper.mappers().getMapper("field")).fieldType(); + + // Create test documents with specific versions + List docs = Arrays.asList( + mapper.parse(source(b -> b.field("field", "1.0.0"))), + mapper.parse(source(b -> b.field("field", "1.1.0"))), + mapper.parse(source(b -> b.field("field", "2.1.0"))) + ); + + Directory dir = new ByteBuffersDirectory(); + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig()); + for (ParsedDocument doc : docs) { + writer.addDocument(doc.rootDoc()); + } + writer.close(); + + IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(dir)); + + Query wildcardQuery = fieldType.wildcardQuery("1.*.0", MultiTermQuery.CONSTANT_SCORE_REWRITE, false, null); + assertThat("Should match 1.0.0 and 1.1.0", searcher.count(wildcardQuery), equalTo(2)); + + dir.close(); + } + + public void testRegexpQuery() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + MappedFieldType fieldType = ((FieldMapper) mapper.mappers().getMapper("field")).fieldType(); + IndexSearcher searcher = setupSearcher(mapper); + + Query regexpQuery = fieldType.regexpQuery( + "1\\.0\\.0-.*", + RegExp.ALL, + 0, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE, + null + ); + assertThat("Should match 1.0.0-alpha and 1.0.0-beta", searcher.count(regexpQuery), equalTo(2)); + } + + public void testFuzzyQuery() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + MappedFieldType fieldType = ((FieldMapper) mapper.mappers().getMapper("field")).fieldType(); + + // Create simple test documents + List docs = Arrays.asList( + mapper.parse(source(b -> b.field("field", "1.0.0"))), + mapper.parse(source(b -> b.field("field", "1.0.1"))) + ); + + Directory dir = new ByteBuffersDirectory(); + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig()); + for (ParsedDocument doc : docs) { + writer.addDocument(doc.rootDoc()); + } + writer.close(); + + IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(dir)); + + // Test fuzzy query + Query fuzzyQuery = fieldType.fuzzyQuery( + "1.0.0", + Fuzziness.ONE, + 0, // No prefix requirement + 50, + true, + MultiTermQuery.CONSTANT_SCORE_REWRITE, + null + ); + + assertThat( + "Fuzzy match should find similar versions", + searcher.count(fuzzyQuery), + equalTo(2) // Should match 1.0.0 and 1.0.1 + ); + + dir.close(); + } + + public void testComplexQuery() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + MappedFieldType fieldType = ((FieldMapper) mapper.mappers().getMapper("field")).fieldType(); + IndexSearcher searcher = setupSearcher(mapper); + + Query complexQuery = new BooleanQuery.Builder().add( + fieldType.prefixQuery("1.", MultiTermQuery.CONSTANT_SCORE_REWRITE, false, null), + BooleanClause.Occur.MUST + ) + .add( + fieldType.regexpQuery( + ".*-.*", + RegExp.ALL, + 0, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE, + null + ), + BooleanClause.Occur.MUST_NOT + ) + .build(); + + assertThat("Should match 1.0.0, 1.0.1, 1.1.0, 1.0.0+build.123", searcher.count(complexQuery), equalTo(4)); + } + + /** + * Test to cover error cases in SemanticVersionFieldType + */ + public void testFieldTypeErrorCases() { + // Create a mock QueryShardContext + QueryShardContext mockContext = mock(QueryShardContext.class); + + // Create a field type with various configurations to test error paths + SemanticVersionFieldMapper.SemanticVersionFieldType fieldType = new SemanticVersionFieldMapper.SemanticVersionFieldType( + "test_field", + new HashMap<>(), + true, // isSearchable + true, // hasDocValues + false // isStored + ); + + // Test termQuery with null value (should throw exception) + IllegalArgumentException nullValueException = expectThrows( + IllegalArgumentException.class, + () -> fieldType.termQuery(null, mockContext) + ); + assertEquals("Cannot search for null value", nullValueException.getMessage()); + + // Create a field type that is not searchable but has doc values + SemanticVersionFieldMapper.SemanticVersionFieldType docValuesOnlyFieldType = + new SemanticVersionFieldMapper.SemanticVersionFieldType( + "docvalues_only", + new HashMap<>(), + false, // isSearchable + true, // hasDocValues + false // isStored + ); + + // Test termQuery - should use doc values query only + Query docValuesQuery = docValuesOnlyFieldType.termQuery("1.0.0", mockContext); + assertNotNull(docValuesQuery); + + // Create a field type that is searchable but has no doc values + SemanticVersionFieldMapper.SemanticVersionFieldType searchOnlyFieldType = new SemanticVersionFieldMapper.SemanticVersionFieldType( + "search_only", + new HashMap<>(), + true, // isSearchable + false, // hasDocValues + false // isStored + ); + + // Test termQuery - should use index query only + Query indexQuery = searchOnlyFieldType.termQuery("1.0.0", mockContext); + assertNotNull(indexQuery); + + // Create a field type that is neither searchable nor has doc values + SemanticVersionFieldMapper.SemanticVersionFieldType invalidFieldType = new SemanticVersionFieldMapper.SemanticVersionFieldType( + "invalid_field", + new HashMap<>(), + false, // isSearchable + false, // hasDocValues + false // isStored + ); + + // Test termQuery - should throw exception + IllegalArgumentException invalidFieldException = expectThrows( + IllegalArgumentException.class, + () -> invalidFieldType.termQuery("1.0.0", mockContext) + ); + assertThat(invalidFieldException.getMessage(), containsString("is neither indexed nor has doc_values enabled")); + + // Test rangeQuery with invalid version format - should throw QueryShardException + QueryShardException rangeException = expectThrows( + QueryShardException.class, + () -> fieldType.rangeQuery("invalid-version", "2.0.0", true, true, null, null, null, mockContext) + ); + assertThat(rangeException.getMessage(), containsString("Failed to create range query for field")); + + // Test termsQuery with different field configurations + List terms = Arrays.asList("1.0.0", "2.0.0", "3.0.0"); + + // Test with searchable field + Query termsQuery = searchOnlyFieldType.termsQuery(terms, mockContext); + assertNotNull(termsQuery); + + // Test with doc values only field + Query docValuesTermsQuery = docValuesOnlyFieldType.termsQuery(terms, mockContext); + assertNotNull(docValuesTermsQuery); + + // Test with invalid field + IllegalArgumentException termsException = expectThrows( + IllegalArgumentException.class, + () -> invalidFieldType.termsQuery(terms, mockContext) + ); + assertThat(termsException.getMessage(), containsString("is neither indexed nor has doc_values enabled")); + + // Test regexpQuery with non-searchable field + IllegalArgumentException regexpException = expectThrows( + IllegalArgumentException.class, + () -> docValuesOnlyFieldType.regexpQuery("1\\.0\\..*", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_REWRITE, mockContext) + ); + assertEquals("Regexp queries require the field to be indexed", regexpException.getMessage()); + + // Test wildcardQuery with case insensitivity + Query wildcardQuery = fieldType.wildcardQuery( + "1.0.*", + MultiTermQuery.CONSTANT_SCORE_REWRITE, + true, // case insensitive + mockContext + ); + assertNotNull(wildcardQuery); + + // Test wildcardQuery with non-searchable field + IllegalArgumentException wildcardException = expectThrows( + IllegalArgumentException.class, + () -> docValuesOnlyFieldType.wildcardQuery("1.0.*", MultiTermQuery.CONSTANT_SCORE_REWRITE, false, mockContext) + ); + assertEquals("Wildcard queries require the field to be indexed", wildcardException.getMessage()); + + // Test prefixQuery with non-searchable field + IllegalArgumentException prefixException = expectThrows( + IllegalArgumentException.class, + () -> docValuesOnlyFieldType.prefixQuery("1.0", MultiTermQuery.CONSTANT_SCORE_REWRITE, false, mockContext) + ); + assertEquals("Prefix queries require the field to be indexed", prefixException.getMessage()); + + // Test fuzzyQuery with null rewrite method (should use default) + Query fuzzyQuery = fieldType.fuzzyQuery( + "1.0.0", + Fuzziness.ONE, + 0, + 50, + true, + null, // null rewrite method + mockContext + ); + assertNotNull(fuzzyQuery); + + // Test fuzzyQuery with non-searchable field + IllegalArgumentException fuzzyException = expectThrows( + IllegalArgumentException.class, + () -> docValuesOnlyFieldType.fuzzyQuery("1.0.0", Fuzziness.ONE, 0, 50, true, MultiTermQuery.CONSTANT_SCORE_REWRITE, mockContext) + ); + assertEquals("Fuzzy queries require the field to be indexed", fuzzyException.getMessage()); + + // Test valueFetcher with format parameter + SearchLookup mockLookup = mock(SearchLookup.class); + IllegalArgumentException formatException = expectThrows( + IllegalArgumentException.class, + () -> fieldType.valueFetcher(mockContext, mockLookup, "some_format") + ); + assertThat(formatException.getMessage(), containsString("doesn't support formats")); + + // Test valueFetcher without format parameter + assertNotNull(fieldType.valueFetcher(mockContext, mockLookup, null)); + + // Test fielddataBuilder with doc_values disabled + IllegalArgumentException fieldDataException = expectThrows( + IllegalArgumentException.class, + () -> searchOnlyFieldType.fielddataBuilder("test_index", null) + ); + assertThat(fieldDataException.getMessage(), containsString("does not have doc_values enabled")); + } +} diff --git a/server/src/test/java/org/opensearch/index/mapper/SemanticVersionTests.java b/server/src/test/java/org/opensearch/index/mapper/SemanticVersionTests.java new file mode 100644 index 0000000000000..f364ea6ff9dde --- /dev/null +++ b/server/src/test/java/org/opensearch/index/mapper/SemanticVersionTests.java @@ -0,0 +1,352 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.mapper; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; + +/** + * Test class for SemanticVersion + */ +public class SemanticVersionTests extends OpenSearchTestCase { + + public void testBasicVersionParsing() { + SemanticVersion version = SemanticVersion.parse("1.2.3"); + assertEquals("1.2.3", version.toString()); + } + + public void testPreReleaseVersionParsing() { + SemanticVersion version = SemanticVersion.parse("1.2.3-alpha"); + assertEquals("1.2.3-alpha", version.toString()); + + version = SemanticVersion.parse("1.2.3-alpha.1"); + assertEquals("1.2.3-alpha.1", version.toString()); + + version = SemanticVersion.parse("1.2.3-0.3.7"); + assertEquals("1.2.3-0.3.7", version.toString()); + + version = SemanticVersion.parse("1.2.3-x.7.z.92"); + assertEquals("1.2.3-x.7.z.92", version.toString()); + } + + public void testBuildMetadataParsing() { + SemanticVersion version = SemanticVersion.parse("1.2.3+build.123"); + assertEquals("1.2.3+build.123", version.toString()); + + version = SemanticVersion.parse("1.2.3+build.123.xyz"); + assertEquals("1.2.3+build.123.xyz", version.toString()); + } + + public void testCompleteVersionParsing() { + SemanticVersion version = SemanticVersion.parse("1.2.3-alpha.1+build.123"); + assertEquals("1.2.3-alpha.1+build.123", version.toString()); + } + + public void testInvalidVersions() { + // Test null + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> SemanticVersion.parse(null)); + assertEquals("Version string cannot be null or empty", e.getMessage()); + + // Test empty string + e = expectThrows(IllegalArgumentException.class, () -> SemanticVersion.parse("")); + assertEquals("Version string cannot be null or empty", e.getMessage()); + + // Test invalid formats + List invalidVersions = Arrays.asList( + "1", + "1.2", + "1.2.3.4", + "1.2.3-", + "1.2.3+", + "01.2.3", + "1.02.3", + "1.2.03", + "1.2.3-@invalid", + "1.2.3+@invalid", + "a.b.c", + "-1.2.3", + "1.-2.3", + "1.2.-3" + ); + + for (String invalid : invalidVersions) { + e = expectThrows(IllegalArgumentException.class, () -> SemanticVersion.parse(invalid)); + assertTrue(e.getMessage().contains("Invalid semantic version format")); + } + } + + public void testNegativeNumbers() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SemanticVersion(-1, 0, 0, null, null)); + assertEquals("Version numbers cannot be negative", e.getMessage()); + } + + public void testVersionComparison() { + // Test major version comparison + assertThat(SemanticVersion.parse("2.0.0").compareTo(SemanticVersion.parse("1.0.0")), greaterThan(0)); + assertThat(SemanticVersion.parse("1.0.0").compareTo(SemanticVersion.parse("2.0.0")), lessThan(0)); + + // Test minor version comparison + assertThat(SemanticVersion.parse("1.2.0").compareTo(SemanticVersion.parse("1.1.0")), greaterThan(0)); + + // Test patch version comparison + assertThat(SemanticVersion.parse("1.1.2").compareTo(SemanticVersion.parse("1.1.1")), greaterThan(0)); + } + + public void testPreReleaseComparison() { + // Pre-release versions have lower precedence + assertThat(SemanticVersion.parse("1.0.0-alpha").compareTo(SemanticVersion.parse("1.0.0")), lessThan(0)); + + // Numeric identifiers + assertThat(SemanticVersion.parse("1.0.0-alpha.1").compareTo(SemanticVersion.parse("1.0.0-alpha.2")), lessThan(0)); + + // Alphanumeric identifiers + assertThat(SemanticVersion.parse("1.0.0-alpha").compareTo(SemanticVersion.parse("1.0.0-beta")), lessThan(0)); + } + + public void testComplexPreReleaseComparison() { + List orderedVersions = Arrays.asList( + "1.0.0-alpha", + "1.0.0-alpha.1", + "1.0.0-alpha.beta", + "1.0.0-beta", + "1.0.0-beta.2", + "1.0.0-beta.11", + "1.0.0-rc.1", + "1.0.0" + ); + + for (int i = 0; i < orderedVersions.size() - 1; i++) { + SemanticVersion v1 = SemanticVersion.parse(orderedVersions.get(i)); + SemanticVersion v2 = SemanticVersion.parse(orderedVersions.get(i + 1)); + assertThat(v1.compareTo(v2), lessThan(0)); + } + } + + public void testBuildMetadataComparison() { + // Build metadata should be ignored in precedence + assertEquals(0, SemanticVersion.parse("1.0.0+build.1").compareTo(SemanticVersion.parse("1.0.0+build.2"))); + assertEquals(0, SemanticVersion.parse("1.0.0").compareTo(SemanticVersion.parse("1.0.0+build"))); + } + + public void testNormalizedString() { + SemanticVersion version = SemanticVersion.parse("1.2.3-alpha+build.123"); + String normalized = version.getNormalizedString(); + + // Check padding + assertTrue(normalized.startsWith("00000000000000000001")); + assertTrue(normalized.contains("00000000000000000002")); + assertTrue(normalized.contains("00000000000000000003")); + + // Check pre-release and build metadata + assertTrue(normalized.contains("-alpha")); + assertTrue(normalized.contains("+build.123")); + } + + public void testEdgeCases() { + // Very large version numbers + SemanticVersion version = SemanticVersion.parse("999999999.999999999.999999999"); + assertEquals("999999999.999999999.999999999", version.toString()); + + // Long pre-release string + version = SemanticVersion.parse("1.0.0-alpha.beta.gamma.delta.epsilon"); + assertEquals("1.0.0-alpha.beta.gamma.delta.epsilon", version.toString()); + + // Long build metadata + version = SemanticVersion.parse("1.0.0+build.123.456.789.abc.def"); + assertEquals("1.0.0+build.123.456.789.abc.def", version.toString()); + } + + public void testNullComparison() { + SemanticVersion version = SemanticVersion.parse("1.0.0"); + assertThat(version.compareTo(null), greaterThan(0)); + } + + public void testPreReleaseIdentifierComparison() { + // Numeric identifiers have lower precedence than non-numeric + assertThat(SemanticVersion.parse("1.0.0-1").compareTo(SemanticVersion.parse("1.0.0-alpha")), lessThan(0)); + + // Longer pre-release version has higher precedence + assertThat(SemanticVersion.parse("1.0.0-alpha").compareTo(SemanticVersion.parse("1.0.0-alpha.1")), lessThan(0)); + } + + public void testGetNormalizedComparableString() { + // Stable release - should end with '~' + SemanticVersion stable = SemanticVersion.parse("1.0.0"); + String stableNorm = stable.getNormalizedComparableString(); + assertTrue(stableNorm.startsWith("00000000000000000001.00000000000000000000.00000000000000000000")); + assertTrue(stableNorm.endsWith("~")); + + // Pre-release alpha - should end with '-alpha' (lowercase) + SemanticVersion alpha = SemanticVersion.parse("1.0.0-alpha"); + String alphaNorm = alpha.getNormalizedComparableString(); + assertTrue(alphaNorm.startsWith("00000000000000000001.00000000000000000000.00000000000000000000")); + assertTrue(alphaNorm.endsWith("-alpha")); + + // Pre-release beta - should end with '-beta' + SemanticVersion beta = SemanticVersion.parse("1.0.0-beta"); + String betaNorm = beta.getNormalizedComparableString(); + assertTrue(betaNorm.startsWith("00000000000000000001.00000000000000000000.00000000000000000000")); + assertTrue(betaNorm.endsWith("-beta")); + + // Pre-release with uppercase (should be lowercased in normalized) + SemanticVersion preReleaseCaps = SemanticVersion.parse("1.0.0-ALPHA.BETA"); + String preReleaseCapsNorm = preReleaseCaps.getNormalizedComparableString(); + assertTrue(preReleaseCapsNorm.endsWith("-alpha.beta")); + + // Stable release with build metadata (build metadata ignored in normalized string) + SemanticVersion stableWithBuild = SemanticVersion.parse("1.0.0+build.123"); + String stableWithBuildNorm = stableWithBuild.getNormalizedComparableString(); + assertEquals(stableNorm, stableWithBuildNorm); + + // Pre-release with build metadata (build metadata ignored) + SemanticVersion preReleaseWithBuild = SemanticVersion.parse("1.0.0-beta+build.456"); + String preReleaseWithBuildNorm = preReleaseWithBuild.getNormalizedComparableString(); + assertEquals(betaNorm, preReleaseWithBuildNorm); + } + + public void testSquareBracketRemoval() { + // Test that square brackets are removed during parsing + SemanticVersion version1 = SemanticVersion.parse("[1.2.3]"); + assertEquals("1.2.3", version1.toString()); + + SemanticVersion version2 = SemanticVersion.parse("[1.2.3-alpha]"); + assertEquals("1.2.3-alpha", version2.toString()); + + SemanticVersion version3 = SemanticVersion.parse("[1.2.3+build.123]"); + assertEquals("1.2.3+build.123", version3.toString()); + + SemanticVersion version4 = SemanticVersion.parse("[1.2.3-alpha+build.123]"); + assertEquals("1.2.3-alpha+build.123", version4.toString()); + } + + public void testWhitespaceHandling() { + // Test that whitespace is converted to dots during parsing + SemanticVersion version1 = SemanticVersion.parse("1 2 3"); + assertEquals("1.2.3", version1.toString()); + + SemanticVersion version2 = SemanticVersion.parse("1 2 3"); + assertEquals("1.2.3", version2.toString()); + + SemanticVersion version3 = SemanticVersion.parse("1\t2\t3"); + assertEquals("1.2.3", version3.toString()); + + SemanticVersion version4 = SemanticVersion.parse("1 2 3"); + assertEquals("1.2.3", version4.toString()); + + // Test mixed whitespace + SemanticVersion version5 = SemanticVersion.parse("1 \t 2 \t 3"); + assertEquals("1.2.3", version5.toString()); + } + + public void testNumberFormatExceptionHandling() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> SemanticVersion.parse("01.02.03")); + assertTrue(e.getMessage().contains("Invalid semantic version format")); + } + + public void testComparePreReleaseMethod() { + // Test numeric vs non-numeric identifiers + assertThat(SemanticVersion.parse("1.0.0-1").compareTo(SemanticVersion.parse("1.0.0-alpha")), lessThan(0)); + assertThat(SemanticVersion.parse("1.0.0-alpha").compareTo(SemanticVersion.parse("1.0.0-1")), greaterThan(0)); + + // Test numeric comparison + assertThat(SemanticVersion.parse("1.0.0-1").compareTo(SemanticVersion.parse("1.0.0-2")), lessThan(0)); + assertThat(SemanticVersion.parse("1.0.0-10").compareTo(SemanticVersion.parse("1.0.0-2")), greaterThan(0)); + + // Test string comparison + assertThat(SemanticVersion.parse("1.0.0-alpha").compareTo(SemanticVersion.parse("1.0.0-beta")), lessThan(0)); + assertThat(SemanticVersion.parse("1.0.0-beta").compareTo(SemanticVersion.parse("1.0.0-alpha")), greaterThan(0)); + + // Test mixed numeric and string + assertThat(SemanticVersion.parse("1.0.0-alpha.1").compareTo(SemanticVersion.parse("1.0.0-alpha.2")), lessThan(0)); + assertThat(SemanticVersion.parse("1.0.0-alpha.2").compareTo(SemanticVersion.parse("1.0.0-alpha.10")), lessThan(0)); + + // Test different lengths + assertThat(SemanticVersion.parse("1.0.0-alpha").compareTo(SemanticVersion.parse("1.0.0-alpha.1")), lessThan(0)); + assertThat(SemanticVersion.parse("1.0.0-alpha.1").compareTo(SemanticVersion.parse("1.0.0-alpha")), greaterThan(0)); + } + + public void testPadWithZerosMethod() { + // Test with small numbers that need padding + SemanticVersion version1 = SemanticVersion.parse("1.2.3"); + String normalized1 = version1.getNormalizedString(); + assertEquals("00000000000000000001.00000000000000000002.00000000000000000003", normalized1); + + // Test with larger numbers that need less padding + SemanticVersion version2 = SemanticVersion.parse("123.456.789"); + String normalized2 = version2.getNormalizedString(); + assertEquals("00000000000000000123.00000000000000000456.00000000000000000789", normalized2); + + // Test with very large numbers + SemanticVersion version3 = SemanticVersion.parse("999999999.999999999.999999999"); + String normalized3 = version3.getNormalizedString(); + assertEquals("00000000000999999999.00000000000999999999.00000000000999999999", normalized3); + + // Test with zero values + SemanticVersion version4 = SemanticVersion.parse("0.0.0"); + String normalized4 = version4.getNormalizedString(); + assertEquals("00000000000000000000.00000000000000000000.00000000000000000000", normalized4); + + // Test with a value that doesn't need padding + String str = "9999999999"; + String padded = SemanticVersion.parse("1.0.0").toString(); + } + + /** + * Test to cover getBuild() method + */ + public void testGetBuild() { + // Test with build metadata + SemanticVersion version = SemanticVersion.parse("1.2.3+build.123"); + assertEquals("build.123", version.getBuild()); + + // Test without build metadata + SemanticVersion versionNoBuild = SemanticVersion.parse("1.2.3"); + assertNull(versionNoBuild.getBuild()); + } + + /** + * Test to cover NumberFormatException handling in parse method + */ + public void testNumberFormatExceptionInParse() { + try { + // This should throw IllegalArgumentException with a NumberFormatException cause + SemanticVersion.parse("2147483648.0.0"); // Integer.MAX_VALUE + 1 + fail("Should have thrown an exception"); + } catch (IllegalArgumentException e) { + // Verify that the exception message contains the expected text + assertTrue(e.getMessage().contains("Invalid version numbers")); + // Verify that the cause is a NumberFormatException + assertTrue(e.getCause() instanceof NumberFormatException); + } + } + + /** + * Test to cover the else branch in comparePreRelease method + */ + public void testComparePreReleaseElseBranch() { + // Create versions with non-numeric pre-release identifiers + SemanticVersion v1 = SemanticVersion.parse("1.0.0-alpha"); + SemanticVersion v2 = SemanticVersion.parse("1.0.0-beta"); + + // alpha comes before beta lexically + assertThat(v1.compareTo(v2), lessThan(0)); + assertThat(v2.compareTo(v1), greaterThan(0)); + + // Test with mixed case to ensure case sensitivity is handled + SemanticVersion v3 = SemanticVersion.parse("1.0.0-Alpha"); + SemanticVersion v4 = SemanticVersion.parse("1.0.0-alpha"); + + // Uppercase comes before lowercase in ASCII + assertThat(v3.compareTo(v4), lessThan(0)); + } +} diff --git a/server/src/test/java/org/opensearch/index/query/SemanticVersionFieldQueryTests.java b/server/src/test/java/org/opensearch/index/query/SemanticVersionFieldQueryTests.java new file mode 100644 index 0000000000000..048ea37aae73c --- /dev/null +++ b/server/src/test/java/org/opensearch/index/query/SemanticVersionFieldQueryTests.java @@ -0,0 +1,419 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.query; + +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.unit.Fuzziness; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.test.OpenSearchSingleNodeTestCase; + +import java.io.IOException; + +import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; + +public class SemanticVersionFieldQueryTests extends OpenSearchSingleNodeTestCase { + + @Override + public void setUp() throws Exception { + super.setUp(); + setupIndex(); + ensureGreen("test"); + } + + protected XContentBuilder createMapping() throws IOException { + return XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("version") + .field("type", "version") + .endObject() + .endObject() + .endObject(); + } + + protected void setupIndex() throws IOException { + XContentBuilder mapping = createMapping(); + client().admin().indices().prepareCreate("test").setMapping(mapping).get(); + ensureGreen(); + + String doc1 = """ + { + "version": "1.0.0" + }"""; + client().index(new IndexRequest("test").id("1").source(doc1, MediaTypeRegistry.JSON).setRefreshPolicy(IMMEDIATE)).actionGet(); + + String doc2 = """ + { + "version": "1.0.1" + }"""; + client().index(new IndexRequest("test").id("2").source(doc2, MediaTypeRegistry.JSON).setRefreshPolicy(IMMEDIATE)).actionGet(); + + String doc3 = """ + { + "version": "1.1.0" + }"""; + client().index(new IndexRequest("test").id("3").source(doc3, MediaTypeRegistry.JSON).setRefreshPolicy(IMMEDIATE)).actionGet(); + + String doc4 = """ + { + "version": "2.0.0" + }"""; + client().index(new IndexRequest("test").id("4").source(doc4, MediaTypeRegistry.JSON).setRefreshPolicy(IMMEDIATE)).actionGet(); + + String doc5 = """ + { + "version": "1.0.0-alpha" + }"""; + client().index(new IndexRequest("test").id("5").source(doc5, MediaTypeRegistry.JSON).setRefreshPolicy(IMMEDIATE)).actionGet(); + + String doc6 = """ + { + "version": "1.0.0-beta" + }"""; + client().index(new IndexRequest("test").id("6").source(doc6, MediaTypeRegistry.JSON).setRefreshPolicy(IMMEDIATE)).actionGet(); + + String doc7 = """ + { + "version": "1.0.0+build.123" + }"""; + client().index(new IndexRequest("test").id("7").source(doc7, MediaTypeRegistry.JSON).setRefreshPolicy(IMMEDIATE)).actionGet(); + } + + /** + * Expected behavior: + * 1.0.0-alpha (Match) + * 1.0.0-beta (Match) + * 1.0.0+build.123 (Match) + * 1.0.0 (Match) + * 1.0.1 (Match) + * 1.1.0 (Match) + * 2.0.0 (Match) + * @throws IOException + */ + public void testExistsQuery() throws IOException { + // Test exists query + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.existsQuery("version")).get(); + + assertSearchResponse(response); + assertHitCount(response, 7); // Should match all documents with a version field + + // Test with non-existent field + response = client().prepareSearch("test").setQuery(QueryBuilders.existsQuery("non_existent_field")).get(); + + assertSearchResponse(response); + assertHitCount(response, 0); + } + + /** + * 1.0.0-alpha (Ignore) + * 1.0.0-beta (Ignore) + * 1.0.0+build.123 (Ignore) + * 1.0.0 (Ignore) + * 1.0.1 (Match) + * 1.1.0 (Match) + * 2.0.0 (Ignore) + * @throws IOException + */ + public void testMatchQuery() throws IOException { + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("version", "1.0.0")).get(); + + assertSearchResponse(response); + assertHitCount(response, 1); // Should match "1.0.0" if analyzed, or 0 if not. + } + + /** + * 1.0.0-alpha (Ignore) + * 1.0.0-beta (Ignore) + * 1.0.0+build.123 (Ignore) + * 1.0.0 (Ignore) + * 1.0.1 (Match) + * 1.1.0 (Match) + * 2.0.0 (Ignore) + * @throws IOException + */ + public void testMultiMatchQuery() throws IOException { + SearchResponse response = client().prepareSearch("test") + .setQuery(QueryBuilders.multiMatchQuery("1.0.0", "version", "non_existent_field")) + .get(); + + assertSearchResponse(response); + assertHitCount(response, 1); // Match only "1.0.0" in "version" + } + + /** + * 1.0.0-alpha (Ignore) + * 1.0.0-beta (Ignore) + * 1.0.0+build.123 (Ignore) + * 1.0.0 (Match) + * 1.0.1 (Ignore) + * 1.1.0 (Ignore) + * 2.0.0 (Ignore) + * @throws IOException if the search request fails + */ + public void testTermQuery() throws IOException { + GetResponse getResponse = client().prepareGet("test", "1").get(); + assertTrue("Document should exist", getResponse.isExists()); + assertEquals("1.0.0", getResponse.getSourceAsMap().get("version")); + + // Term query + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.termQuery("version", "1.0.0")).get(); + + assertSearchResponse(response); + assertHitCount(response, 1); + } + + /** + * 1.0.0-alpha (Ignore) + * 1.0.0-beta (Ignore) + * 1.0.0+build.123 (Ignore) + * 1.0.0 (Match) + * 1.0.1 (Match) + * 1.1.0 (Ignore) + * 2.0.0 (Ignore) + * @throws IOException + */ + public void testTermsQuery() throws IOException { + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.termsQuery("version", "1.0.0", "1.0.1")).get(); + + assertSearchResponse(response); + assertHitCount(response, 2); + } + + /** + * 1.0.0-alpha (Ignore) + * 1.0.0-beta (Ignore) + * 1.0.0+build.123 (Ignore) + * 1.0.0 (Ignore) + * 1.0.1 (Match) + * 1.1.0 (Match) + * 2.0.0 (Ignore) + * @throws IOException + */ + public void testRangeQuery() throws IOException { + SearchResponse response = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("version").gt("1.0.0").lt("2.0.0")) + .get(); + assertSearchResponse(response); + assertHitCount(response, 2); + } + + /** + * 1.0.0-alpha (Match) + * 1.0.0-beta (Match) + * 1.0.0+build.123 (Match) + * 1.0.0 (Match) + * 1.0.1 (Ignore) + * 1.1.0 (Ignore) + * 2.0.0 (Ignore) + * @throws IOException + */ + public void testRangeQueryIncludingPreRelease() throws IOException { + SearchResponse response = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("version").gte("1.0.0-alpha").lt("1.0.1")) + .get(); + + assertSearchResponse(response); + assertHitCount(response, 4); + } + + /** + * 1.0.0-alpha (Match) + * 1.0.0-beta (Match) + * 1.0.0+build.123 (Match) + * 1.0.0 (Match) + * 1.0.1 (Match) + * 1.1.0 (Match) + * 2.0.0 (Ignore) + * @throws IOException + */ + public void testPrefixQuery() throws IOException { + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.prefixQuery("version", "1.")).get(); + assertSearchResponse(response); + assertHitCount(response, 6); + } + + /** + * 1.0.0-alpha (Ignore) + * 1.0.0-beta (Ignore) + * 1.0.0+build.123 (Ignore) + * 1.0.0 (Match) + * 1.0.1 (Ignore) + * 1.1.0 (Ignore) + * 2.0.0 (Match) + * @throws IOException + */ + public void testWildcardQuery() throws IOException { + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.wildcardQuery("version", "*.0.0")).get(); + assertSearchResponse(response); + assertHitCount(response, 2); + } + + /** + * 1.0.0-alpha (Ignore) + * 1.0.0-beta (Ignore) + * 1.0.0+build.123 (Ignore) + * 1.0.0 (Match) + * 1.0.1 (Ignore) + * 1.1.0 (Match) + * 2.0.0 (Ignore) + * @throws IOException + */ + public void testRegexpQuery() throws IOException { + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.regexpQuery("version", "1\\..*\\.0")).get(); + assertSearchResponse(response); + assertHitCount(response, 2); + } + + /** + * 1.0.0-alpha (Ignore) + * 1.0.0-beta (Ignore) + * 1.0.0+build.123 (Ignore) + * 1.0.0 (Match) + * 1.0.1 (Match) + * 1.1.0 (Ignore) + * 2.0.0 (Ignore) + * @throws IOException + */ + public void testFuzzyQuery() throws IOException { + SearchResponse response = client().prepareSearch("test") + .setQuery(QueryBuilders.fuzzyQuery("version", "1.0.1").fuzziness(Fuzziness.ONE)) + .get(); + assertSearchResponse(response); + assertHitCount(response, 2); + } + + /** + * 1.0.0-alpha — excluded because of dash (mustNot) + * 1.0.0-beta — excluded (mustNot) + * 1.0.0+build.123 — no dash, starts with 1. → included + * 1.0.0 — included + * 1.0.1 — included + * 1.1.0 — included (starts with 1. and no dash) + * 2.0.0 — excluded (does not start with 1.) + * @throws IOException + */ + public void testComplexQuery() throws IOException { + SearchResponse response = client().prepareSearch("test") + .setQuery( + QueryBuilders.boolQuery() + .must(QueryBuilders.prefixQuery("version", "1.")) + .mustNot(QueryBuilders.regexpQuery("version", ".*-.*")) + .should(QueryBuilders.termQuery("version", "1.0.0")) + ) + .get(); + assertSearchResponse(response); + assertHitCount(response, 4); + } + + /** + * Should match documents with IDs 1 and 2 regardless of version: + * Document 1 - version 1.0.0 + * Document 2 - version 1.0.1 + * @throws IOException + */ + public void testIdsQuery() throws IOException { + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.idsQuery().addIds("1", "2")).get(); + + assertSearchResponse(response); + assertHitCount(response, 2); + } + + /** + * 1.0.0-alpha (Ignore) + * 1.0.0-beta (Ignore) + * 1.0.0+build.123 (Ignore) + * 1.0.0 (Match) + * 1.0.1 (Ignore) + * 1.1.0 (Ignore) + * 2.0.0 (Ignore) + * @throws IOException + */ + public void testConstantScoreQuery() throws IOException { + SearchResponse response = client().prepareSearch("test") + .setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery("version", "1.0.0"))) + .get(); + + assertSearchResponse(response); + assertHitCount(response, 1); + } + + /** + * 1.0.0-alpha (Ignore) + * 1.0.0-beta (Ignore) + * 1.0.0+build.123 (Ignore) + * 1.0.0 (Match) + * 1.0.1 (Ignore) + * 1.1.0 (Ignore) + * 2.0.0 (Ignore) + * @throws IOException + */ + public void testFunctionScoreQuery() throws IOException { + SearchResponse response = client().prepareSearch("test") + .setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("version", "1.0.0"))) + .get(); + + assertSearchResponse(response); + assertHitCount(response, 1); + } + + /** + * 1.0.0-alpha (Ignore) + * 1.0.0-beta (Ignore) + * 1.0.0+build.123 (Ignore) + * 1.0.0 (Ignore) + * 1.0.1 (Match) + * 1.1.0 (Match) + * 2.0.0 (Ignore) + * @throws IOException + */ + public void testMatchPhraseQuery() throws IOException { + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.matchPhraseQuery("version", "1.0.0")).get(); + + assertSearchResponse(response); + assertHitCount(response, 1); + } + + /** + * Sorted ascending: + * First hit: 1.0.0-alpha + * Second hit: 1.0.0-beta + * Third hit: 1.0.0 + * Fourth hit: 1.0.0+build.123 + * Fifth hit: 1.0.1 + * Sixth hit: 1.1.0 + * Seventh hit: 2.0.0 + * @throws IOException + */ + public void testSortByVersion() throws IOException { + SearchResponse response = client().prepareSearch("test") + .addSort("version", org.opensearch.search.sort.SortOrder.ASC) + .setQuery(QueryBuilders.matchAllQuery()) + .get(); + + assertSearchResponse(response); + assertHitCount(response, 7); + assertEquals("1.0.0-alpha", response.getHits().getAt(0).getSourceAsMap().get("version")); + } + + /** + * Test that a term query for an invalid version returns no hits. + * @throws IOException + */ + public void testTermQueryInvalidVersion() throws IOException { + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.termQuery("version", "invalid-version")).get(); + + assertSearchResponse(response); + assertHitCount(response, 0); + } +} diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java index d9348aacd7a11..fc92065391fd4 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java @@ -120,6 +120,7 @@ import org.opensearch.index.mapper.ObjectMapper.Nested; import org.opensearch.index.mapper.RangeFieldMapper; import org.opensearch.index.mapper.RangeType; +import org.opensearch.index.mapper.SemanticVersionFieldMapper; import org.opensearch.index.mapper.StarTreeMapper; import org.opensearch.index.mapper.TextFieldMapper; import org.opensearch.index.query.QueryBuilder; @@ -213,6 +214,7 @@ public abstract class AggregatorTestCase extends OpenSearchTestCase { denylist.add(FieldAliasMapper.CONTENT_TYPE); // TODO support alias denylist.add(DerivedFieldMapper.CONTENT_TYPE); // TODO support derived fields denylist.add(StarTreeMapper.CONTENT_TYPE); // TODO evaluate support for star tree fields + denylist.add(SemanticVersionFieldMapper.CONTENT_TYPE); // TODO support for semantic version fields TYPE_TEST_DENYLIST = denylist; } From 621421e79d7a97cc1cbd9c0fb3f2c03fa5fe3928 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Jul 2025 07:46:19 -0400 Subject: [PATCH 25/51] Bump com.azure:azure-core from 1.55.3 to 1.55.5 in /plugins/repository-azure (#18691) * Bump com.azure:azure-core in /plugins/repository-azure Bumps [com.azure:azure-core](https://github.com/Azure/azure-sdk-for-java) from 1.55.3 to 1.55.5. - [Release notes](https://github.com/Azure/azure-sdk-for-java/releases) - [Commits](https://github.com/Azure/azure-sdk-for-java/compare/azure-core_1.55.3...azure-core_1.55.5) --- updated-dependencies: - dependency-name: com.azure:azure-core dependency-version: 1.55.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- plugins/repository-azure/licenses/azure-core-1.55.3.jar.sha1 | 1 - plugins/repository-azure/licenses/azure-core-1.55.5.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/azure-core-1.55.3.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-core-1.55.5.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 15d826dd6c20a..16f8bb2bf9367 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.bouncycastle:bcpkix-fips` from 2.0.7 to 2.0.8 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) - Bump `org.bouncycastle:bcpg-fips` from 2.0.10 to 2.0.11 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) - Bump `com.password4j:password4j` from 1.8.2 to 1.8.3 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) +- Bump `com.azure:azure-core` from 1.55.3 to 1.55.5 ([#18691](https://github.com/opensearch-project/OpenSearch/pull/18691)) ### Deprecated diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 2946e5a5d8a11..3576f28a57c32 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -44,7 +44,7 @@ opensearchplugin { } dependencies { - api 'com.azure:azure-core:1.55.3' + api 'com.azure:azure-core:1.55.5' api 'com.azure:azure-json:1.5.0' api 'com.azure:azure-xml:1.2.0' api 'com.azure:azure-storage-common:12.29.0' diff --git a/plugins/repository-azure/licenses/azure-core-1.55.3.jar.sha1 b/plugins/repository-azure/licenses/azure-core-1.55.3.jar.sha1 deleted file mode 100644 index 966919b5c3c86..0000000000000 --- a/plugins/repository-azure/licenses/azure-core-1.55.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -120adc6c3de019097b163390a7eb511f0acd050b \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-core-1.55.5.jar.sha1 b/plugins/repository-azure/licenses/azure-core-1.55.5.jar.sha1 new file mode 100644 index 0000000000000..da66667656d04 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-core-1.55.5.jar.sha1 @@ -0,0 +1 @@ +93227034496e2a0dc0b7babcbba57f5a6bb8b4cb \ No newline at end of file From efc3e5dfde2c2479ff5c86a3963b22dd81844658 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Wed, 9 Jul 2025 06:09:28 +0530 Subject: [PATCH 26/51] Exposing default codec from codec service (#18698) * Exposing default codec from codec service Signed-off-by: bharath-techie * addressing comments Signed-off-by: bharath-techie --------- Signed-off-by: bharath-techie --- .../main/java/org/opensearch/index/codec/CodecService.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index 1568e262fb6b9..dcd681ac2bf0b 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -93,6 +93,13 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe this.codecs = codecs.immutableMap(); } + /** + * Returns default codec + */ + public final Codec defaultCodec() { + return codecs.get(DEFAULT_CODEC); + } + public Codec codec(String name) { Codec codec = codecs.get(name); if (codec == null) { From 26ada7ee02a7209a0af13290f281e2a4f644520c Mon Sep 17 00:00:00 2001 From: Harsh Kothari Date: Wed, 9 Jul 2025 11:22:48 +0530 Subject: [PATCH 27/51] Replace CPU load average logic with AverageTracker class and modify default thresholds (#18666) Signed-off-by: Harsh Kothari --- CHANGELOG.md | 1 + .../AutoForceMergeManagerIT.java | 17 +-- .../autoforcemerge/AutoForceMergeManager.java | 120 +++++++++++------- .../ForceMergeManagerSettings.java | 12 +- .../ResourceTrackerProvider.java | 85 +++++++++++++ .../AutoForceMergeManagerTests.java | 62 ++++++--- .../ForceMergeManagerSettingsTests.java | 9 +- 7 files changed, 222 insertions(+), 84 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/autoforcemerge/ResourceTrackerProvider.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 16f8bb2bf9367..c0460505faf46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix the visit of sub queries for HasParentQuery and HasChildQuery ([#18621](https://github.com/opensearch-project/OpenSearch/pull/18621)) - Fix the backward compatibility regression with COMPLEMENT for Regexp queries introduced in OpenSearch 3.0 ([#18640](https://github.com/opensearch-project/OpenSearch/pull/18640)) - Fix Replication lag computation ([#18602](https://github.com/opensearch-project/OpenSearch/pull/18602)) +- Fixed Staggered merge - load average replace with AverageTrackers, some Default thresholds modified ([#18666](https://github.com/opensearch-project/OpenSearch/pull/18666)) ### Security diff --git a/server/src/internalClusterTest/java/org/opensearch/index/autoforcemerge/AutoForceMergeManagerIT.java b/server/src/internalClusterTest/java/org/opensearch/index/autoforcemerge/AutoForceMergeManagerIT.java index bd17d82c4d46e..142e2da95653e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/autoforcemerge/AutoForceMergeManagerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/autoforcemerge/AutoForceMergeManagerIT.java @@ -45,11 +45,6 @@ public class AutoForceMergeManagerIT extends RemoteStoreBaseIntegTestCase { private static final String MERGE_DELAY = "1s"; private static final Integer SEGMENT_COUNT = 1; - @Override - protected boolean addMockIndexStorePlugin() { - return false; - } - @Override protected Settings nodeSettings(int nodeOrdinal) { ByteSizeValue cacheSize = new ByteSizeValue(16, ByteSizeUnit.GB); @@ -158,8 +153,8 @@ public void testAutoForceMergeTriggeringBasicWithOneShard() throws Exception { SegmentsStats segmentsStatsBefore = shard.segmentStats(false, false); waitUntil(() -> shard.segmentStats(false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES); SegmentsStats segmentsStatsAfter = shard.segmentStats(false, false); - // assertTrue((int) segmentsStatsBefore.getCount() > segmentsStatsAfter.getCount()); - // assertEquals((int) SEGMENT_COUNT, segmentsStatsAfter.getCount()); + assertTrue((int) segmentsStatsBefore.getCount() > segmentsStatsAfter.getCount()); + assertEquals((int) SEGMENT_COUNT, segmentsStatsAfter.getCount()); assertAcked(client().admin().indices().prepareDelete(INDEX_NAME_1).get()); } @@ -221,11 +216,11 @@ public void testAutoForceMergeTriggeringBasicWithFiveShardsOfTwoIndex() throws E SegmentsStats segmentsStatsForShard3Before = shard3.segmentStats(false, false); SegmentsStats segmentsStatsForShard4Before = shard4.segmentStats(false, false); SegmentsStats segmentsStatsForShard5Before = shard5.segmentStats(false, false); - AtomicLong totalSegments = new AtomicLong( + AtomicLong totalSegmentsBefore = new AtomicLong( segmentsStatsForShard1Before.getCount() + segmentsStatsForShard2Before.getCount() + segmentsStatsForShard3Before.getCount() + segmentsStatsForShard4Before.getCount() + segmentsStatsForShard5Before.getCount() ); - assertTrue(totalSegments.get() > 5); + assertTrue(totalSegmentsBefore.get() > 5); waitUntil(() -> shard1.segmentStats(false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES); waitUntil(() -> shard2.segmentStats(false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES); waitUntil(() -> shard3.segmentStats(false, false).getCount() == SEGMENT_COUNT, 1, TimeUnit.MINUTES); @@ -236,11 +231,11 @@ public void testAutoForceMergeTriggeringBasicWithFiveShardsOfTwoIndex() throws E SegmentsStats segmentsStatsForShard3After = shard3.segmentStats(false, false); SegmentsStats segmentsStatsForShard4After = shard4.segmentStats(false, false); SegmentsStats segmentsStatsForShard5After = shard5.segmentStats(false, false); - totalSegments.set( + AtomicLong totalSegmentsAfter = new AtomicLong( segmentsStatsForShard1After.getCount() + segmentsStatsForShard2After.getCount() + segmentsStatsForShard3After.getCount() + segmentsStatsForShard4After.getCount() + segmentsStatsForShard5After.getCount() ); - // assertEquals(5, totalSegments.get()); + assertTrue(totalSegmentsBefore.get() > totalSegmentsAfter.get()); assertAcked(client().admin().indices().prepareDelete(INDEX_NAME_1).get()); assertAcked(client().admin().indices().prepareDelete(INDEX_NAME_2).get()); } diff --git a/server/src/main/java/org/opensearch/index/autoforcemerge/AutoForceMergeManager.java b/server/src/main/java/org/opensearch/index/autoforcemerge/AutoForceMergeManager.java index f79b0e72c683a..ae7f98a138407 100644 --- a/server/src/main/java/org/opensearch/index/autoforcemerge/AutoForceMergeManager.java +++ b/server/src/main/java/org/opensearch/index/autoforcemerge/AutoForceMergeManager.java @@ -66,10 +66,11 @@ public class AutoForceMergeManager extends AbstractLifecycleComponent { private ConfigurationValidator configurationValidator; private NodeValidator nodeValidator; private ShardValidator shardValidator; + private Integer allocatedProcessors; + private ResourceTrackerProvider.ResourceTrackers resourceTrackers; private final ForceMergeManagerSettings forceMergeManagerSettings; private final CommonStatsFlags flags = new CommonStatsFlags(CommonStatsFlags.Flag.Segments, CommonStatsFlags.Flag.Translog); private final Set mergingShards; - private Integer allocatedProcessors; private static final Logger logger = LogManager.getLogger(AutoForceMergeManager.class); @@ -96,6 +97,7 @@ protected void doStart() { this.nodeValidator = new NodeValidator(); this.shardValidator = new ShardValidator(); this.allocatedProcessors = OpenSearchExecutors.allocatedProcessors(clusterService.getSettings()); + this.resourceTrackers = ResourceTrackerProvider.create(threadPool); } @Override @@ -117,43 +119,65 @@ private void modifySchedulerInterval(TimeValue schedulerInterval) { } private void triggerForceMerge() { + if (isValidForForceMerge() == false) { + return; + } + executeForceMergeOnShards(); + } + + private boolean isValidForForceMerge() { if (configurationValidator.hasWarmNodes() == false) { + resourceTrackers.stop(); logger.debug("No warm nodes found. Skipping Auto Force merge."); - return; + return false; } if (nodeValidator.validate().isAllowed() == false) { logger.debug("Node capacity constraints are not allowing to trigger auto ForceMerge"); - return; + return false; } - int iteration = nodeValidator.getMaxConcurrentForceMerges(); + return true; + } + + private void executeForceMergeOnShards() { + int remainingIterations = nodeValidator.getMaxConcurrentForceMerges(); for (IndexShard shard : getShardsBasedOnSorting(indicesService)) { - if (iteration == 0) { + if (remainingIterations == 0 || !nodeValidator.validate().isAllowed()) { + if (remainingIterations > 0) { + logger.debug("Node conditions no longer suitable for force merge."); + } break; } - if (nodeValidator.validate().isAllowed() == false) { - logger.debug("Node conditions no longer suitable for force merge."); + remainingIterations--; + executeForceMergeForShard(shard); + if (!waitBetweenShards()) { break; } - iteration--; - CompletableFuture.runAsync(() -> { - try { - mergingShards.add(shard.shardId().getId()); - shard.forceMerge(new ForceMergeRequest().maxNumSegments(forceMergeManagerSettings.getSegmentCount())); - logger.debug("Merging is completed successfully for the shard {}", shard.shardId()); - } catch (Exception e) { - logger.error("Error during force merge for shard {}\nException: {}", shard.shardId(), e); - } finally { - mergingShards.remove(shard.shardId().getId()); - } - }, threadPool.executor(ThreadPool.Names.FORCE_MERGE)); - logger.info("Successfully triggered force merge for shard {}", shard.shardId()); + } + } + + private void executeForceMergeForShard(IndexShard shard) { + CompletableFuture.runAsync(() -> { try { - Thread.sleep(forceMergeManagerSettings.getForcemergeDelay().getMillis()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.error("Timer was interrupted while waiting between shards", e); - break; + mergingShards.add(shard.shardId().getId()); + shard.forceMerge(new ForceMergeRequest().maxNumSegments(forceMergeManagerSettings.getSegmentCount())); + logger.debug("Merging is completed successfully for the shard {}", shard.shardId()); + } catch (Exception e) { + logger.error("Error during force merge for shard {}\nException: {}", shard.shardId(), e); + } finally { + mergingShards.remove(shard.shardId().getId()); } + }, threadPool.executor(ThreadPool.Names.FORCE_MERGE)); + logger.info("Successfully triggered force merge for shard {}", shard.shardId()); + } + + private boolean waitBetweenShards() { + try { + Thread.sleep(forceMergeManagerSettings.getForcemergeDelay().getMillis()); + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error("Timer was interrupted while waiting between shards", e); + return false; } } @@ -264,15 +288,14 @@ protected class NodeValidator implements ValidationStrategy { @Override public ValidationResult validate() { + resourceTrackers.start(); if (isCpuUsageOverThreshold()) { return new ValidationResult(false); } if (isDiskUsageOverThreshold()) { return new ValidationResult(false); } - double jvmUsedPercent = jvmService.stats().getMem().getHeapUsedPercent(); - if (jvmUsedPercent >= forceMergeManagerSettings.getJvmThreshold()) { - logger.debug("JVM memory: {}% breached the threshold: {}", jvmUsedPercent, forceMergeManagerSettings.getJvmThreshold()); + if (isJvmUsageOverThreshold()) { return new ValidationResult(false); } if (areForceMergeThreadsAvailable() == false) { @@ -291,24 +314,34 @@ private boolean areForceMergeThreadsAvailable() { return false; } + private boolean isJvmUsageOverThreshold() { + double jvmAverage = resourceTrackers.jvmFiveMinute.getAverage(); + if (jvmAverage >= forceMergeManagerSettings.getJvmThreshold()) { + logger.debug("JVM Average: 5m({}%) breached the threshold: {}", jvmAverage, forceMergeManagerSettings.getJvmThreshold()); + return true; + } + jvmAverage = resourceTrackers.jvmOneMinute.getAverage(); + if (jvmAverage >= forceMergeManagerSettings.getJvmThreshold()) { + logger.debug("JVM Average: 1m({}%) breached the threshold: {}", jvmAverage, forceMergeManagerSettings.getJvmThreshold()); + return true; + } + double jvmUsedPercent = jvmService.stats().getMem().getHeapUsedPercent(); + if (jvmUsedPercent >= forceMergeManagerSettings.getJvmThreshold()) { + logger.debug("JVM memory: {}% breached the threshold: {}", jvmUsedPercent, forceMergeManagerSettings.getJvmThreshold()); + return true; + } + return false; + } + private boolean isCpuUsageOverThreshold() { - double[] loadAverage = osService.stats().getCpu().getLoadAverage(); - double loadAverage5m = (loadAverage[1] / (double) allocatedProcessors) * 100; - if (loadAverage5m >= forceMergeManagerSettings.getCpuThreshold()) { - logger.debug( - "Load Average: 5m({}%) breached the threshold: {}", - loadAverage5m, - forceMergeManagerSettings.getCpuThreshold() - ); + double cpuAverage = resourceTrackers.cpuFiveMinute.getAverage(); + if (cpuAverage >= forceMergeManagerSettings.getCpuThreshold()) { + logger.debug("CPU Average: 5m({}%) breached the threshold: {}", cpuAverage, forceMergeManagerSettings.getCpuThreshold()); return true; } - double loadAverage1m = (loadAverage[0] / (double) allocatedProcessors) * 100; - if (loadAverage1m >= forceMergeManagerSettings.getCpuThreshold()) { - logger.debug( - "Load Average: 1m({}%) breached the threshold: {}", - loadAverage1m, - forceMergeManagerSettings.getCpuThreshold() - ); + cpuAverage = resourceTrackers.cpuOneMinute.getAverage(); + if (cpuAverage >= forceMergeManagerSettings.getCpuThreshold()) { + logger.debug("CPU Average: 1m({}%) breached the threshold: {}", cpuAverage, forceMergeManagerSettings.getCpuThreshold()); return true; } double cpuPercent = osService.stats().getCpu().getPercent(); @@ -445,6 +478,7 @@ protected boolean mustReschedule() { @Override protected void runInternal() { if (configurationValidator.validate().isAllowed() == false) { + resourceTrackers.stop(); return; } triggerForceMerge(); diff --git a/server/src/main/java/org/opensearch/index/autoforcemerge/ForceMergeManagerSettings.java b/server/src/main/java/org/opensearch/index/autoforcemerge/ForceMergeManagerSettings.java index b1d9ccc77988c..4077cd5768574 100644 --- a/server/src/main/java/org/opensearch/index/autoforcemerge/ForceMergeManagerSettings.java +++ b/server/src/main/java/org/opensearch/index/autoforcemerge/ForceMergeManagerSettings.java @@ -56,11 +56,11 @@ public class ForceMergeManagerSettings { ); /** - * Setting for wait time between force merge operations (default: 10s). + * Setting for wait time between force merge operations (default: 15s). */ public static final Setting MERGE_DELAY_BETWEEN_SHARDS_FOR_AUTO_FORCE_MERGE = Setting.timeSetting( "node.auto_force_merge.merge_delay", - TimeValue.timeValueSeconds(10), + TimeValue.timeValueSeconds(15), TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(60), Setting.Property.Dynamic, @@ -92,11 +92,11 @@ public class ForceMergeManagerSettings { ); /** - * Setting for cpu threshold. (default: 80) + * Setting for cpu threshold. (default: 75) */ public static final Setting CPU_THRESHOLD_PERCENTAGE_FOR_AUTO_FORCE_MERGE = Setting.doubleSetting( "node.auto_force_merge.cpu.threshold", - 80.0, + 75.0, 10, 100, Setting.Property.Dynamic, @@ -104,11 +104,11 @@ public class ForceMergeManagerSettings { ); /** - * Setting for memory threshold. (default: 90) + * Setting for disk threshold. (default: 85) */ public static final Setting DISK_THRESHOLD_PERCENTAGE_FOR_AUTO_FORCE_MERGE = Setting.doubleSetting( "node.auto_force_merge.disk.threshold", - 90.0, + 85.0, 10, 100, Setting.Property.Dynamic, diff --git a/server/src/main/java/org/opensearch/index/autoforcemerge/ResourceTrackerProvider.java b/server/src/main/java/org/opensearch/index/autoforcemerge/ResourceTrackerProvider.java new file mode 100644 index 0000000000000..6db76a6359ea9 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/autoforcemerge/ResourceTrackerProvider.java @@ -0,0 +1,85 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.autoforcemerge; + +import org.opensearch.common.unit.TimeValue; +import org.opensearch.node.resource.tracker.AverageCpuUsageTracker; +import org.opensearch.node.resource.tracker.AverageMemoryUsageTracker; +import org.opensearch.threadpool.ThreadPool; + +/** + * Provider for creating resource usage trackers used in auto force merge operations. + * + * @opensearch.internal + */ +public class ResourceTrackerProvider { + + public static final TimeValue SHORT_POLL_INTERVAL = TimeValue.timeValueSeconds(6); + public static final TimeValue LONG_POLL_INTERVAL = TimeValue.timeValueSeconds(30); + public static final TimeValue SHORT_AVERAGE_WINDOW = TimeValue.timeValueMinutes(1); + public static final TimeValue LONG_AVERAGE_WINDOW = TimeValue.timeValueMinutes(5); + + public static ResourceTrackers resourceTrackers; + + public static ResourceTrackers create(ThreadPool threadPool) { + return resourceTrackers = new ResourceTrackers( + new AverageCpuUsageTracker(threadPool, SHORT_POLL_INTERVAL, SHORT_AVERAGE_WINDOW), + new AverageCpuUsageTracker(threadPool, LONG_POLL_INTERVAL, LONG_AVERAGE_WINDOW), + new AverageMemoryUsageTracker(threadPool, SHORT_POLL_INTERVAL, SHORT_AVERAGE_WINDOW), + new AverageMemoryUsageTracker(threadPool, LONG_POLL_INTERVAL, LONG_AVERAGE_WINDOW) + ); + } + + /** + * Container for resource usage trackers used in auto force merge operations. + * Provides access to CPU and JVM memory usage trackers with different time windows. + * + * @opensearch.internal + */ + public static class ResourceTrackers { + public final AverageCpuUsageTracker cpuOneMinute; + public final AverageCpuUsageTracker cpuFiveMinute; + public final AverageMemoryUsageTracker jvmOneMinute; + public final AverageMemoryUsageTracker jvmFiveMinute; + + /** + * Creates a new ResourceTrackers instance. + * + * @param cpuOneMinute CPU tracker with 1-minute window + * @param cpuFiveMinute CPU tracker with 5-minute window + * @param jvmOneMinute JVM memory tracker with 1-minute window + * @param jvmFiveMinute JVM memory tracker with 5-minute window + */ + ResourceTrackers( + AverageCpuUsageTracker cpuOneMinute, + AverageCpuUsageTracker cpuFiveMinute, + AverageMemoryUsageTracker jvmOneMinute, + AverageMemoryUsageTracker jvmFiveMinute + ) { + this.cpuOneMinute = cpuOneMinute; + this.cpuFiveMinute = cpuFiveMinute; + this.jvmOneMinute = jvmOneMinute; + this.jvmFiveMinute = jvmFiveMinute; + } + + public void start() { + cpuOneMinute.start(); + cpuFiveMinute.start(); + jvmOneMinute.start(); + jvmFiveMinute.start(); + } + + public void stop() { + cpuOneMinute.stop(); + cpuFiveMinute.stop(); + jvmOneMinute.stop(); + jvmFiveMinute.stop(); + } + } +} diff --git a/server/src/test/java/org/opensearch/index/autoforcemerge/AutoForceMergeManagerTests.java b/server/src/test/java/org/opensearch/index/autoforcemerge/AutoForceMergeManagerTests.java index 5a8d7fc16ad3e..8f33c3534423a 100644 --- a/server/src/test/java/org/opensearch/index/autoforcemerge/AutoForceMergeManagerTests.java +++ b/server/src/test/java/org/opensearch/index/autoforcemerge/AutoForceMergeManagerTests.java @@ -18,6 +18,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.lifecycle.Lifecycle; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -176,7 +177,6 @@ public void testConfigurationValidatorWithNonDataNode() { // NodeValidator Tests public void testNodeValidatorWithHealthyResources() { when(cpu.getPercent()).thenReturn((short) 50); - when(cpu.getLoadAverage()).thenReturn(new double[]{0.7 * allocatedProcessors, 0.6 * allocatedProcessors, 0.5 * allocatedProcessors}); when(jvm.getHeapUsedPercent()).thenReturn((short) 60); ThreadPoolStats stats = new ThreadPoolStats( Arrays.asList(new ThreadPoolStats.Stats( @@ -191,9 +191,32 @@ public void testNodeValidatorWithHealthyResources() { autoForceMergeManager.close(); } + public void testNodeValidatorWithFeatureSwitch() { + when(cpu.getPercent()).thenReturn((short) 50); + when(jvm.getHeapUsedPercent()).thenReturn((short) 60); + ThreadPoolStats stats = new ThreadPoolStats( + Arrays.asList(new ThreadPoolStats.Stats( + ThreadPool.Names.FORCE_MERGE, 1, 0, 0, 0, 1, 0, 0 + )) + ); + when(threadPool.stats()).thenReturn(stats); + Settings settings = getConfiguredClusterSettings(false, false, Collections.emptyMap()); + AutoForceMergeManager autoForceMergeManager = clusterSetupWithNode(settings, getNodeWithRoles(DATA_NODE_1, Set.of(DiscoveryNodeRole.DATA_ROLE))); + autoForceMergeManager.start(); + assertFalse(autoForceMergeManager.getConfigurationValidator().validate().isAllowed()); + assertNotEquals(Lifecycle.State.STARTED, ResourceTrackerProvider.resourceTrackers.cpuFiveMinute.lifecycleState()); + assertNotEquals(Lifecycle.State.STARTED, ResourceTrackerProvider.resourceTrackers.cpuFiveMinute.lifecycleState()); + assertNotEquals(Lifecycle.State.STARTED, ResourceTrackerProvider.resourceTrackers.cpuFiveMinute.lifecycleState()); + assertNotEquals(Lifecycle.State.STARTED, ResourceTrackerProvider.resourceTrackers.cpuFiveMinute.lifecycleState()); + assertTrue(autoForceMergeManager.getNodeValidator().validate().isAllowed()); + assertEquals(Lifecycle.State.STARTED, ResourceTrackerProvider.resourceTrackers.cpuFiveMinute.lifecycleState()); + assertEquals(Lifecycle.State.STARTED, ResourceTrackerProvider.resourceTrackers.cpuFiveMinute.lifecycleState()); + assertEquals(Lifecycle.State.STARTED, ResourceTrackerProvider.resourceTrackers.cpuFiveMinute.lifecycleState()); + assertEquals(Lifecycle.State.STARTED, ResourceTrackerProvider.resourceTrackers.cpuFiveMinute.lifecycleState()); + autoForceMergeManager.close(); + } + public void testNodeValidatorWithHighCPU() { - when(cpu.getPercent()).thenReturn((short) 95); - when(cpu.getLoadAverage()).thenReturn(new double[]{0.7 * allocatedProcessors, 0.6 * allocatedProcessors, 0.5 * allocatedProcessors}); DiscoveryNode dataNode1 = getNodeWithRoles(DATA_NODE_1, Set.of(DiscoveryNodeRole.DATA_ROLE)); DiscoveryNode warmNode1 = getNodeWithRoles(WARM_NODE_1, Set.of(DiscoveryNodeRole.WARM_ROLE)); ClusterState clusterState = ClusterState.builder(new ClusterName(ClusterServiceUtils.class.getSimpleName())) @@ -207,20 +230,24 @@ public void testNodeValidatorWithHighCPU() { .blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK) .build(); when(clusterService.state()).thenReturn(clusterState); - AutoForceMergeManager autoForceMergeManager = clusterSetupWithNode(getConfiguredClusterSettings(true, true, Collections.emptyMap()), dataNode1); + AutoForceMergeManager autoForceMergeManager = clusterSetupWithNode( + getConfiguredClusterSettings(true, true, Collections.emptyMap()), + dataNode1 + ); autoForceMergeManager.start(); + when(cpu.getPercent()).thenReturn((short) 95); assertFalse(autoForceMergeManager.getNodeValidator().validate().isAllowed()); - when(cpu.getPercent()).thenReturn((short) 50); - when(cpu.getLoadAverage()).thenReturn(new double[]{0.9 * allocatedProcessors, 0.6 * allocatedProcessors, 0.5 * allocatedProcessors}); + for (int i = 0; i < 10; i++) + ResourceTrackerProvider.resourceTrackers.cpuOneMinute.recordUsage(90); assertFalse(autoForceMergeManager.getNodeValidator().validate().isAllowed()); - when(cpu.getLoadAverage()).thenReturn(new double[]{0.7 * allocatedProcessors, 0.9 * allocatedProcessors, 0.5 * allocatedProcessors}); + for (int i = 0; i < 10; i++) + ResourceTrackerProvider.resourceTrackers.cpuFiveMinute.recordUsage(90); assertFalse(autoForceMergeManager.getNodeValidator().validate().isAllowed()); autoForceMergeManager.close(); } public void testNodeValidatorWithHighDiskUsage() { when(cpu.getPercent()).thenReturn((short) 50); - when(cpu.getLoadAverage()).thenReturn(new double[]{0.7 * allocatedProcessors, 0.6 * allocatedProcessors, 0.5 * allocatedProcessors}); when(disk.getAvailable()).thenReturn(new ByteSizeValue(5)); AutoForceMergeManager autoForceMergeManager = clusterSetupWithNode(getConfiguredClusterSettings(true, true, Collections.emptyMap()), getNodeWithRoles(DATA_NODE_1, Set.of(DiscoveryNodeRole.DATA_ROLE))); autoForceMergeManager.start(); @@ -230,17 +257,21 @@ public void testNodeValidatorWithHighDiskUsage() { public void testNodeValidatorWithHighJVMUsage() { when(cpu.getPercent()).thenReturn((short) 50); - when(cpu.getLoadAverage()).thenReturn(new double[]{0.7 * allocatedProcessors, 0.6 * allocatedProcessors, 0.5 * allocatedProcessors}); - when(jvm.getHeapUsedPercent()).thenReturn((short) 90); AutoForceMergeManager autoForceMergeManager = clusterSetupWithNode(getConfiguredClusterSettings(true, true, Collections.emptyMap()), getNodeWithRoles(DATA_NODE_1, Set.of(DiscoveryNodeRole.DATA_ROLE))); autoForceMergeManager.start(); + when(jvm.getHeapUsedPercent()).thenReturn((short) 90); + assertFalse(autoForceMergeManager.getNodeValidator().validate().isAllowed()); + for(int i = 0; i < 10; i++) + ResourceTrackerProvider.resourceTrackers.jvmOneMinute.recordUsage(90); + assertFalse(autoForceMergeManager.getNodeValidator().validate().isAllowed()); + for(int i = 0; i < 10; i++) + ResourceTrackerProvider.resourceTrackers.jvmFiveMinute.recordUsage(90); assertFalse(autoForceMergeManager.getNodeValidator().validate().isAllowed()); autoForceMergeManager.close(); } public void testNodeValidatorWithInsufficientForceMergeThreads() { when(cpu.getPercent()).thenReturn((short) 50); - when(cpu.getLoadAverage()).thenReturn(new double[]{0.7 * allocatedProcessors, 0.6 * allocatedProcessors, 0.5 * allocatedProcessors}); when(jvm.getHeapUsedPercent()).thenReturn((short) 50); ThreadPoolStats stats = new ThreadPoolStats( Arrays.asList(new ThreadPoolStats.Stats( @@ -376,9 +407,6 @@ public void testForceMergeOperationOnDataNodeWithFailingMerges() throws IOExcept .build(); when(clusterService.state()).thenReturn(clusterState); when(cpu.getPercent()).thenReturn((short) 50); - when(cpu.getLoadAverage()).thenReturn( - new double[] { 0.7 * allocatedProcessors, 0.6 * allocatedProcessors, 0.5 * allocatedProcessors } - ); when(jvm.getHeapUsedPercent()).thenReturn((short) 50); int forceMergeThreads = 4; @@ -430,9 +458,6 @@ public void testForceMergeOperationOnDataNodeOfWarmEnabledCluster() throws IOExc .build(); when(clusterService.state()).thenReturn(clusterState); when(cpu.getPercent()).thenReturn((short) 50); - when(cpu.getLoadAverage()).thenReturn( - new double[] { 0.7 * allocatedProcessors, 0.6 * allocatedProcessors, 0.5 * allocatedProcessors } - ); when(jvm.getHeapUsedPercent()).thenReturn((short) 50); int forceMergeThreads = 4; ExecutorService executorService = Executors.newFixedThreadPool(forceMergeThreads); @@ -488,9 +513,6 @@ public void testForceMergeOperationOnDataNodeWithThreadInterruption() throws Int .build(); when(clusterService.state()).thenReturn(clusterState); when(cpu.getPercent()).thenReturn((short) 50); - when(cpu.getLoadAverage()).thenReturn( - new double[] { 0.7 * allocatedProcessors, 0.6 * allocatedProcessors, 0.5 * allocatedProcessors } - ); when(jvm.getHeapUsedPercent()).thenReturn((short) 50); int forceMergeThreads = 4; diff --git a/server/src/test/java/org/opensearch/index/autoforcemerge/ForceMergeManagerSettingsTests.java b/server/src/test/java/org/opensearch/index/autoforcemerge/ForceMergeManagerSettingsTests.java index cb3c008f0de08..7aded4023670a 100644 --- a/server/src/test/java/org/opensearch/index/autoforcemerge/ForceMergeManagerSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/autoforcemerge/ForceMergeManagerSettingsTests.java @@ -61,12 +61,13 @@ public void accept(TimeValue timeValue) { public void testDefaultSettings() { assertEquals(false, forceMergeManagerSettings.isAutoForceMergeFeatureEnabled()); - assertEquals(forceMergeManagerSettings.getForcemergeDelay(), TimeValue.timeValueSeconds(10)); + assertEquals(forceMergeManagerSettings.getForcemergeDelay(), TimeValue.timeValueSeconds(15)); assertEquals(forceMergeManagerSettings.getSchedulerInterval(), TimeValue.timeValueMinutes(30)); assertEquals(2, (int) forceMergeManagerSettings.getConcurrencyMultiplier()); assertEquals(1, (int) forceMergeManagerSettings.getSegmentCount()); - assertEquals(80.0, forceMergeManagerSettings.getCpuThreshold(), 0.0); + assertEquals(75.0, forceMergeManagerSettings.getCpuThreshold(), 0.0); assertEquals(75.0, forceMergeManagerSettings.getJvmThreshold(), 0.0); + assertEquals(85.0, forceMergeManagerSettings.getDiskThreshold(), 0.0); } public void testDynamicSettingsUpdate() { @@ -100,14 +101,14 @@ public void testTimeValueSettings() { Settings newSettings = Settings.builder() .put(ForceMergeManagerSettings.AUTO_FORCE_MERGE_SCHEDULER_INTERVAL.getKey(), "10m") .put(ForceMergeManagerSettings.TRANSLOG_AGE_AUTO_FORCE_MERGE.getKey(), "10m") - .put(ForceMergeManagerSettings.MERGE_DELAY_BETWEEN_SHARDS_FOR_AUTO_FORCE_MERGE.getKey(), "15s") + .put(ForceMergeManagerSettings.MERGE_DELAY_BETWEEN_SHARDS_FOR_AUTO_FORCE_MERGE.getKey(), "20s") .build(); clusterSettings.applySettings(newSettings); assertEquals(forceMergeManagerSettings.getSchedulerInterval(), TimeValue.timeValueMinutes(10)); assertEquals(forceMergeManagerSettings.getTranslogAge(), TimeValue.timeValueMinutes(10)); - assertEquals(forceMergeManagerSettings.getForcemergeDelay(), TimeValue.timeValueSeconds(15)); + assertEquals(forceMergeManagerSettings.getForcemergeDelay(), TimeValue.timeValueSeconds(20)); } public void testThreadSettings() { From a2e7d75920bd283cd87822fe3a6eac5e39be5cb2 Mon Sep 17 00:00:00 2001 From: Ruirui Zhang Date: Tue, 8 Jul 2025 23:57:04 -0700 Subject: [PATCH 28/51] Rename all query group references to workload group (#18711) Signed-off-by: Ruirui Zhang --- CHANGELOG.md | 1 + .../org/opensearch/rule/autotagging/Rule.java | 2 +- .../plugin/wlm/action/package-info.java | 2 +- .../plugin/wlm/rest/package-info.java | 2 +- .../WorkloadGroupPersistenceService.java | 8 ++--- .../plugin/wlm/service/package-info.java | 2 +- .../wlm/WorkloadManagementPluginTests.java | 2 +- .../WorkloadGroupPersistenceServiceTests.java | 8 ++--- .../wlm/MutableWorkloadGroupFragment.java | 2 +- .../opensearch/wlm/WorkloadGroupService.java | 32 +++++++++---------- .../wlm/WorkloadGroupsStateAccessor.java | 4 +-- .../wlm/WorkloadManagementSettings.java | 4 +-- .../wlm/stats/WorkloadGroupState.java | 22 ++++++------- .../wlm/stats/WorkloadGroupStats.java | 2 +- .../opensearch/wlm/stats/package-info.java | 2 +- .../wlm/tracker/CpuUsageCalculator.java | 2 +- .../wlm/tracker/MemoryUsageCalculator.java | 2 +- .../wlm/tracker/ResourceUsageCalculator.java | 6 ++-- .../wlm/WorkloadGroupServiceTests.java | 2 +- ...orkloadGroupTaskResourceTrackingTests.java | 2 +- 20 files changed, 55 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0460505faf46..3c52950c82739 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add support for custom index name resolver from cluster plugin ([#18593](https://github.com/opensearch-project/OpenSearch/pull/18593)) - Disallow resize for Warm Index, add Parameterized ITs for close in remote store ([#18686](https://github.com/opensearch-project/OpenSearch/pull/18686)) - Ability to run Code Coverage with Gradle and produce the jacoco reports locally ([#18509](https://github.com/opensearch-project/OpenSearch/issues/18509)) +- [Workload Management] Update logging and Javadoc, rename QueryGroup to WorkloadGroup ([#18711](https://github.com/opensearch-project/OpenSearch/issues/18711)) - Add NodeResourceUsageStats to ClusterInfo ([#18480](https://github.com/opensearch-project/OpenSearch/issues/18472)) - Introduce SecureHttpTransportParameters experimental API (to complement SecureTransportParameters counterpart) ([#18572](https://github.com/opensearch-project/OpenSearch/issues/18572)) - Create equivalents of JSM's AccessController in the java agent ([#18346](https://github.com/opensearch-project/OpenSearch/issues/18346)) diff --git a/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/autotagging/Rule.java b/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/autotagging/Rule.java index ae1cebdda99d5..5907b9aebecc9 100644 --- a/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/autotagging/Rule.java +++ b/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/autotagging/Rule.java @@ -31,7 +31,7 @@ * of a rule. The indexed view may differ in representation. * { * "id": "fwehf8302582mglfio349==", - * "description": "Assign Query Group for Index Logs123" + * "description": "Assign Workload Group for Index Logs123" * "index_pattern": ["logs123"], * "workload_group": "dev_workload_group_id", * "updated_at": "01-10-2025T21:23:21.456Z" diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/package-info.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/package-info.java index df30f55a99b3c..5cd8892949436 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/package-info.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/package-info.java @@ -7,6 +7,6 @@ */ /** - * Package for the action classes related to query groups in WorkloadManagementPlugin + * Package for the action classes related to workload groups in WorkloadManagementPlugin */ package org.opensearch.plugin.wlm.action; diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/package-info.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/package-info.java index 889f3e107db07..abefac4fe9675 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/package-info.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/package-info.java @@ -7,6 +7,6 @@ */ /** - * Package for the rest classes related to query groups in WorkloadManagementPlugin + * Package for the rest classes related to workload groups in WorkloadManagementPlugin */ package org.opensearch.plugin.wlm.rest; diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/WorkloadGroupPersistenceService.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/WorkloadGroupPersistenceService.java index 8fd5fe5dfcfed..f37e90509c0fb 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/WorkloadGroupPersistenceService.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/WorkloadGroupPersistenceService.java @@ -54,7 +54,7 @@ public class WorkloadGroupPersistenceService { /** * max WorkloadGroup count setting name */ - public static final String QUERY_GROUP_COUNT_SETTING_NAME = "node.workload_group.max_count"; + public static final String WORKLOAD_GROUP_COUNT_SETTING_NAME = "node.workload_group.max_count"; /** * default max workloadGroup count on any node at any given point in time */ @@ -67,7 +67,7 @@ public class WorkloadGroupPersistenceService { * max WorkloadGroup count setting */ public static final Setting MAX_QUERY_GROUP_COUNT = Setting.intSetting( - QUERY_GROUP_COUNT_SETTING_NAME, + WORKLOAD_GROUP_COUNT_SETTING_NAME, DEFAULT_MAX_QUERY_GROUP_COUNT_VALUE, 0, WorkloadGroupPersistenceService::validateMaxWorkloadGroupCount, @@ -116,7 +116,7 @@ public void setMaxWorkloadGroupCount(int newMaxWorkloadGroupCount) { */ private static void validateMaxWorkloadGroupCount(int maxWorkloadGroupCount) { if (maxWorkloadGroupCount > DEFAULT_MAX_QUERY_GROUP_COUNT_VALUE || maxWorkloadGroupCount < MIN_QUERY_GROUP_COUNT_VALUE) { - throw new IllegalArgumentException(QUERY_GROUP_COUNT_SETTING_NAME + " should be in range [1-100]."); + throw new IllegalArgumentException(WORKLOAD_GROUP_COUNT_SETTING_NAME + " should be in range [1-100]."); } } @@ -162,7 +162,7 @@ ClusterState saveWorkloadGroupInClusterState(final WorkloadGroup workloadGroup, // check if maxWorkloadGroupCount will breach if (existingWorkloadGroups.size() == maxWorkloadGroupCount) { - logger.warn("{} value exceeded its assigned limit of {}.", QUERY_GROUP_COUNT_SETTING_NAME, maxWorkloadGroupCount); + logger.warn("{} value exceeded its assigned limit of {}.", WORKLOAD_GROUP_COUNT_SETTING_NAME, maxWorkloadGroupCount); throw new IllegalStateException("Can't create more than " + maxWorkloadGroupCount + " WorkloadGroups in the system."); } diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/package-info.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/package-info.java index e8c88ee656dc7..ac01ed9fb4923 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/package-info.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/package-info.java @@ -7,6 +7,6 @@ */ /** - * Package for the service classes related to query groups in WorkloadManagementPlugin + * Package for the service classes related to workload groups in WorkloadManagementPlugin */ package org.opensearch.plugin.wlm.service; diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementPluginTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementPluginTests.java index 9de5c94a24bf3..57bb1e33d2373 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementPluginTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementPluginTests.java @@ -138,7 +138,7 @@ public void testGetFeatureTypeReturnsWorkloadGroupFeatureType() { assertEquals("workload_group", featureType.getName()); } - public void testGetSettingsIncludesMaxQueryGroupCount() { + public void testGetSettingsIncludesMaxWorkloadGroupCount() { List settings = plugin.getSettings(); assertTrue(settings.contains(WorkloadGroupPersistenceService.MAX_QUERY_GROUP_COUNT)); } diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/WorkloadGroupPersistenceServiceTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/WorkloadGroupPersistenceServiceTests.java index 571103b32205d..e6ea2fc8a2700 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/WorkloadGroupPersistenceServiceTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/WorkloadGroupPersistenceServiceTests.java @@ -59,8 +59,8 @@ import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.workloadGroupPersistenceService; import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.workloadGroupTwo; import static org.opensearch.plugin.wlm.action.WorkloadGroupActionTestUtils.updateWorkloadGroupRequest; -import static org.opensearch.plugin.wlm.service.WorkloadGroupPersistenceService.QUERY_GROUP_COUNT_SETTING_NAME; import static org.opensearch.plugin.wlm.service.WorkloadGroupPersistenceService.SOURCE; +import static org.opensearch.plugin.wlm.service.WorkloadGroupPersistenceService.WORKLOAD_GROUP_COUNT_SETTING_NAME; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.anyString; @@ -155,7 +155,7 @@ public void testCreateWorkloadGroupOverflowCount() { .updatedAt(1690934400000L) .build(); Metadata metadata = Metadata.builder().workloadGroups(Map.of(_ID_ONE, workloadGroupOne, _ID_TWO, workloadGroupTwo)).build(); - Settings settings = Settings.builder().put(QUERY_GROUP_COUNT_SETTING_NAME, 2).build(); + Settings settings = Settings.builder().put(WORKLOAD_GROUP_COUNT_SETTING_NAME, 2).build(); ClusterSettings clusterSettings = new ClusterSettings(settings, clusterSettingsSet()); ClusterService clusterService = new ClusterService(settings, clusterSettings, mock(ThreadPool.class)); ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build(); @@ -174,7 +174,7 @@ public void testCreateWorkloadGroupOverflowCount() { * Tests the invalid value of {@code node.workload_group.max_count} */ public void testInvalidMaxWorkloadGroupCount() { - Settings settings = Settings.builder().put(QUERY_GROUP_COUNT_SETTING_NAME, 2).build(); + Settings settings = Settings.builder().put(WORKLOAD_GROUP_COUNT_SETTING_NAME, 2).build(); ClusterSettings clusterSettings = new ClusterSettings(settings, clusterSettingsSet()); ClusterService clusterService = new ClusterService(settings, clusterSettings, mock(ThreadPool.class)); WorkloadGroupPersistenceService workloadGroupPersistenceService = new WorkloadGroupPersistenceService( @@ -189,7 +189,7 @@ public void testInvalidMaxWorkloadGroupCount() { * Tests the valid value of {@code node.workload_group.max_count} */ public void testValidMaxSandboxCountSetting() { - Settings settings = Settings.builder().put(QUERY_GROUP_COUNT_SETTING_NAME, 100).build(); + Settings settings = Settings.builder().put(WORKLOAD_GROUP_COUNT_SETTING_NAME, 100).build(); ClusterService clusterService = new ClusterService(settings, clusterSettings(), mock(ThreadPool.class)); WorkloadGroupPersistenceService workloadGroupPersistenceService = new WorkloadGroupPersistenceService( clusterService, diff --git a/server/src/main/java/org/opensearch/wlm/MutableWorkloadGroupFragment.java b/server/src/main/java/org/opensearch/wlm/MutableWorkloadGroupFragment.java index 329659b891f9f..d09f486225091 100644 --- a/server/src/main/java/org/opensearch/wlm/MutableWorkloadGroupFragment.java +++ b/server/src/main/java/org/opensearch/wlm/MutableWorkloadGroupFragment.java @@ -185,7 +185,7 @@ public Map getResourceLimits() { /** * This enum models the different WorkloadGroup resiliency modes - * SOFT - means that this query group can consume more than query group resource limits if node is not in duress + * SOFT - means that this workload group can consume more than workload group resource limits if node is not in duress * ENFORCED - means that it will never breach the assigned limits and will cancel as soon as the limits are breached * MONITOR - it will not cause any cancellation but just log the eligible task cancellations */ diff --git a/server/src/main/java/org/opensearch/wlm/WorkloadGroupService.java b/server/src/main/java/org/opensearch/wlm/WorkloadGroupService.java index 970844a2b59b0..86ef0973762db 100644 --- a/server/src/main/java/org/opensearch/wlm/WorkloadGroupService.java +++ b/server/src/main/java/org/opensearch/wlm/WorkloadGroupService.java @@ -139,7 +139,7 @@ protected void doStart() { try { doRun(); } catch (Exception e) { - logger.debug("Exception occurred in Query Sandbox service", e); + logger.debug("Exception occurred in Workload Group service", e); } }, this.workloadManagementSettings.getWorkloadGroupServiceRunInterval(), ThreadPool.Names.GENERIC); } @@ -160,26 +160,26 @@ public void clusterChanged(ClusterChangedEvent event) { Metadata previousMetadata = event.previousState().metadata(); Metadata currentMetadata = event.state().metadata(); - // Extract the query groups from both the current and previous cluster states + // Extract the workload groups from both the current and previous cluster states Map previousWorkloadGroups = previousMetadata.workloadGroups(); Map currentWorkloadGroups = currentMetadata.workloadGroups(); - // Detect new query groups added in the current cluster state + // Detect new workload groups added in the current cluster state for (String workloadGroupName : currentWorkloadGroups.keySet()) { if (!previousWorkloadGroups.containsKey(workloadGroupName)) { - // New query group detected + // New workload group detected WorkloadGroup newWorkloadGroup = currentWorkloadGroups.get(workloadGroupName); - // Perform any necessary actions with the new query group + // Perform any necessary actions with the new workload group workloadGroupsStateAccessor.addNewWorkloadGroup(newWorkloadGroup.get_id()); } } - // Detect query groups deleted in the current cluster state + // Detect workload groups deleted in the current cluster state for (String workloadGroupName : previousWorkloadGroups.keySet()) { if (!currentWorkloadGroups.containsKey(workloadGroupName)) { - // Query group deleted + // Workload group deleted WorkloadGroup deletedWorkloadGroup = previousWorkloadGroups.get(workloadGroupName); - // Perform any necessary actions with the deleted query group + // Perform any necessary actions with the deleted workload group this.deletedWorkloadGroups.add(deletedWorkloadGroup); workloadGroupsStateAccessor.removeWorkloadGroup(deletedWorkloadGroup.get_id()); } @@ -188,13 +188,13 @@ public void clusterChanged(ClusterChangedEvent event) { } /** - * updates the failure stats for the query group + * updates the failure stats for the workload group * - * @param workloadGroupId query group identifier + * @param workloadGroupId workload group identifier */ public void incrementFailuresFor(final String workloadGroupId) { WorkloadGroupState workloadGroupState = workloadGroupsStateAccessor.getWorkloadGroupState(workloadGroupId); - // This can happen if the request failed for a deleted query group + // This can happen if the request failed for a deleted workload group // or new workloadGroup is being created and has not been acknowledged yet if (workloadGroupState == null) { return; @@ -203,7 +203,7 @@ public void incrementFailuresFor(final String workloadGroupId) { } /** - * @return node level query group stats + * @return node level workload group stats */ public WorkloadGroupStats nodeStats(Set workloadGroupIds, Boolean requestedBreached) { final Map statsHolderMap = new HashMap<>(); @@ -250,7 +250,7 @@ public boolean resourceLimitBreached(String id, WorkloadGroupState currentState) } /** - * @param workloadGroupId query group identifier + * @param workloadGroupId workload group identifier */ public void rejectIfNeeded(String workloadGroupId) { if (workloadManagementSettings.getWlmMode() != WlmMode.ENABLED) { @@ -260,8 +260,8 @@ public void rejectIfNeeded(String workloadGroupId) { if (workloadGroupId == null || workloadGroupId.equals(WorkloadGroupTask.DEFAULT_WORKLOAD_GROUP_ID_SUPPLIER.get())) return; WorkloadGroupState workloadGroupState = workloadGroupsStateAccessor.getWorkloadGroupState(workloadGroupId); - // This can happen if the request failed for a deleted query group - // or new workloadGroup is being created and has not been acknowledged yet or invalid query group id + // This can happen if the request failed for a deleted workload group + // or new workloadGroup is being created and has not been acknowledged yet or invalid workload group id if (workloadGroupState == null) { return; } @@ -348,7 +348,7 @@ public void onTaskCompleted(Task task) { final WorkloadGroupTask workloadGroupTask = (WorkloadGroupTask) task; String workloadGroupId = workloadGroupTask.getWorkloadGroupId(); - // set the default workloadGroupId if not existing in the active query groups + // set the default workloadGroupId if not existing in the active workload groups String finalWorkloadGroupId = workloadGroupId; boolean exists = activeWorkloadGroups.stream().anyMatch(workloadGroup -> workloadGroup.get_id().equals(finalWorkloadGroupId)); diff --git a/server/src/main/java/org/opensearch/wlm/WorkloadGroupsStateAccessor.java b/server/src/main/java/org/opensearch/wlm/WorkloadGroupsStateAccessor.java index 103871d1c7fe0..582730bbf0b33 100644 --- a/server/src/main/java/org/opensearch/wlm/WorkloadGroupsStateAccessor.java +++ b/server/src/main/java/org/opensearch/wlm/WorkloadGroupsStateAccessor.java @@ -31,7 +31,7 @@ public WorkloadGroupsStateAccessor(Map workloadGroup } /** - * returns the query groups state + * returns the workload groups state */ public Map getWorkloadGroupStateMap() { return workloadGroupStateMap; @@ -40,7 +40,7 @@ public Map getWorkloadGroupStateMap() { /** * return WorkloadGroupState for the given workloadGroupId * @param workloadGroupId - * @return WorkloadGroupState for the given workloadGroupId, if id is invalid return default query group state + * @return WorkloadGroupState for the given workloadGroupId, if id is invalid return default workload group state */ public WorkloadGroupState getWorkloadGroupState(String workloadGroupId) { return workloadGroupStateMap.getOrDefault( diff --git a/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java b/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java index 35a043a6e42d8..e2a6b3538802c 100644 --- a/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java +++ b/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java @@ -49,13 +49,13 @@ public class WorkloadManagementSettings { ); /** - * Setting name for Query Group Service run interval + * Setting name for Workload Group Service run interval */ public static final String QUERYGROUP_ENFORCEMENT_INTERVAL_SETTING_NAME = "wlm.workload_group.enforcement_interval"; private TimeValue workloadGroupServiceRunInterval; /** - * Setting to control the run interval of Query Group Service + * Setting to control the run interval of Workload Group Service */ public static final Setting QUERYGROUP_SERVICE_RUN_INTERVAL_SETTING = Setting.longSetting( QUERYGROUP_ENFORCEMENT_INTERVAL_SETTING_NAME, diff --git a/server/src/main/java/org/opensearch/wlm/stats/WorkloadGroupState.java b/server/src/main/java/org/opensearch/wlm/stats/WorkloadGroupState.java index 78cb4521aff18..a3715eb72f385 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/WorkloadGroupState.java +++ b/server/src/main/java/org/opensearch/wlm/stats/WorkloadGroupState.java @@ -15,26 +15,26 @@ import java.util.Map; /** - * This class will keep the point in time view of the query group stats + * This class will keep the point in time view of the workload group stats */ public class WorkloadGroupState { /** - * co-ordinator level completions at the query group level, this is a cumulative counter since the Opensearch start time + * co-ordinator level completions at the workload group level, this is a cumulative counter since the Opensearch start time */ public final CounterMetric totalCompletions = new CounterMetric(); /** - * rejections at the query group level, this is a cumulative counter since the OpenSearch start time + * rejections at the workload group level, this is a cumulative counter since the OpenSearch start time */ public final CounterMetric totalRejections = new CounterMetric(); /** - * this will track the cumulative failures in a query group + * this will track the cumulative failures in a workload group */ public final CounterMetric failures = new CounterMetric(); /** - * This will track total number of cancellations in the query group due to all resource type breaches + * This will track total number of cancellations in the workload group due to all resource type breaches */ public final CounterMetric totalCancellations = new CounterMetric(); @@ -54,7 +54,7 @@ public WorkloadGroupState() { /** * - * @return co-ordinator completions in the query group + * @return co-ordinator completions in the workload group */ public long getTotalCompletions() { return totalCompletions.count(); @@ -62,7 +62,7 @@ public long getTotalCompletions() { /** * - * @return rejections in the query group + * @return rejections in the workload group */ public long getTotalRejections() { return totalRejections.count(); @@ -70,7 +70,7 @@ public long getTotalRejections() { /** * - * @return failures in the query group + * @return failures in the workload group */ public long getFailures() { return failures.count(); @@ -81,15 +81,15 @@ public long getTotalCancellations() { } /** - * getter for query group resource state - * @return the query group resource state + * getter for workload group resource state + * @return the workload group resource state */ public Map getResourceState() { return resourceState; } /** - * This class holds the resource level stats for the query group + * This class holds the resource level stats for the workload group */ public static class ResourceTypeState { public final ResourceType resourceType; diff --git a/server/src/main/java/org/opensearch/wlm/stats/WorkloadGroupStats.java b/server/src/main/java/org/opensearch/wlm/stats/WorkloadGroupStats.java index baec9eee980ea..1174424ed398e 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/WorkloadGroupStats.java +++ b/server/src/main/java/org/opensearch/wlm/stats/WorkloadGroupStats.java @@ -87,7 +87,7 @@ public Map getStats() { } /** - * This is a stats holder object which will hold the data for a query group at a point in time + * This is a stats holder object which will hold the data for a workload group at a point in time * the instance will only be created on demand through stats api */ public static class WorkloadGroupStatsHolder implements ToXContentObject, Writeable { diff --git a/server/src/main/java/org/opensearch/wlm/stats/package-info.java b/server/src/main/java/org/opensearch/wlm/stats/package-info.java index 2facf8d16df22..780ba6b33aa50 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/package-info.java +++ b/server/src/main/java/org/opensearch/wlm/stats/package-info.java @@ -7,6 +7,6 @@ */ /** - * Query group stats related artifacts + * Workload group stats related artifacts */ package org.opensearch.wlm.stats; diff --git a/server/src/main/java/org/opensearch/wlm/tracker/CpuUsageCalculator.java b/server/src/main/java/org/opensearch/wlm/tracker/CpuUsageCalculator.java index 32fc0487e4e66..01f2d1071259d 100644 --- a/server/src/main/java/org/opensearch/wlm/tracker/CpuUsageCalculator.java +++ b/server/src/main/java/org/opensearch/wlm/tracker/CpuUsageCalculator.java @@ -14,7 +14,7 @@ import java.util.List; /** - * class to help make cpu usage calculations for the query group + * class to help make cpu usage calculations for the workload group */ public class CpuUsageCalculator extends ResourceUsageCalculator { // This value should be initialised at the start time of the process and be used throughout the codebase diff --git a/server/src/main/java/org/opensearch/wlm/tracker/MemoryUsageCalculator.java b/server/src/main/java/org/opensearch/wlm/tracker/MemoryUsageCalculator.java index 6edb011f399e3..a03487f568228 100644 --- a/server/src/main/java/org/opensearch/wlm/tracker/MemoryUsageCalculator.java +++ b/server/src/main/java/org/opensearch/wlm/tracker/MemoryUsageCalculator.java @@ -15,7 +15,7 @@ import java.util.List; /** - * class to help make memory usage calculations for the query group + * class to help make memory usage calculations for the workload group */ public class MemoryUsageCalculator extends ResourceUsageCalculator { public static final long HEAP_SIZE_BYTES = JvmStats.jvmStats().getMem().getHeapMax().getBytes(); diff --git a/server/src/main/java/org/opensearch/wlm/tracker/ResourceUsageCalculator.java b/server/src/main/java/org/opensearch/wlm/tracker/ResourceUsageCalculator.java index 7ec1f531c8920..6766ca57c38d1 100644 --- a/server/src/main/java/org/opensearch/wlm/tracker/ResourceUsageCalculator.java +++ b/server/src/main/java/org/opensearch/wlm/tracker/ResourceUsageCalculator.java @@ -14,14 +14,14 @@ import java.util.List; /** - * This class is used to track query group level resource usage + * This class is used to track workload group level resource usage */ @PublicApi(since = "2.18.0") public abstract class ResourceUsageCalculator { /** - * calculates the current resource usage for the query group + * calculates the current resource usage for the workload group * - * @param tasks list of tasks in the query group + * @param tasks list of tasks in the workload group */ public abstract double calculateResourceUsage(List tasks); diff --git a/server/src/test/java/org/opensearch/wlm/WorkloadGroupServiceTests.java b/server/src/test/java/org/opensearch/wlm/WorkloadGroupServiceTests.java index 4784fe6cedf5a..989d390853cc2 100644 --- a/server/src/test/java/org/opensearch/wlm/WorkloadGroupServiceTests.java +++ b/server/src/test/java/org/opensearch/wlm/WorkloadGroupServiceTests.java @@ -477,7 +477,7 @@ public void testShouldSBPHandle() { task.setWorkloadGroupId(mockThreadPool.getThreadContext()); assertTrue(workloadGroupService.shouldSBPHandle(task)); - // Valid query group task but wlm not enabled + // Valid workload group task but wlm not enabled when(mockWorkloadManagementSettings.getWlmMode()).thenReturn(WlmMode.DISABLED); activeWorkloadGroups.add( new WorkloadGroup( diff --git a/server/src/test/java/org/opensearch/wlm/tracker/WorkloadGroupTaskResourceTrackingTests.java b/server/src/test/java/org/opensearch/wlm/tracker/WorkloadGroupTaskResourceTrackingTests.java index 123352c6f67a5..ed8f296cd39f0 100644 --- a/server/src/test/java/org/opensearch/wlm/tracker/WorkloadGroupTaskResourceTrackingTests.java +++ b/server/src/test/java/org/opensearch/wlm/tracker/WorkloadGroupTaskResourceTrackingTests.java @@ -50,7 +50,7 @@ public void testValidWorkloadGroupTasksCase() { WorkloadGroupTask task = new SearchTask(1, "test", "test", () -> "Test", TaskId.EMPTY_TASK_ID, new HashMap<>()); taskResourceTrackingService.startTracking(task); - // since the query group id is not set we should not track this task + // since the workload group id is not set we should not track this task Map resourceUsageViewMap = workloadGroupResourceUsageTrackerService .constructWorkloadGroupLevelUsageViews(); assertTrue(resourceUsageViewMap.isEmpty()); From f1825fd45bd56e368391994f1310e1d18231e1b3 Mon Sep 17 00:00:00 2001 From: Ruirui Zhang Date: Wed, 9 Jul 2025 00:16:18 -0700 Subject: [PATCH 29/51] Rename WorkloadGroupTestUtil to WorkloadManagementTestUtil (#18709) Signed-off-by: Ruirui Zhang --- CHANGELOG.md | 1 + ....java => WorkloadManagementTestUtils.java} | 2 +- .../CreateWorkloadGroupRequestTests.java | 4 +- .../CreateWorkloadGroupResponseTests.java | 8 +-- .../DeleteWorkloadGroupRequestTests.java | 6 +- .../action/GetWorkloadGroupRequestTests.java | 6 +- .../action/GetWorkloadGroupResponseTests.java | 18 +++--- .../TransportGetWorkloadGroupActionTests.java | 6 +- .../UpdateWorkloadGroupRequestTests.java | 4 +- .../UpdateWorkloadGroupResponseTests.java | 6 +- .../RestDeleteWorkloadGroupActionTests.java | 2 +- .../WorkloadGroupPersistenceServiceTests.java | 60 +++++++++---------- 12 files changed, 62 insertions(+), 61 deletions(-) rename plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/{WorkloadGroupTestUtils.java => WorkloadManagementTestUtils.java} (99%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c52950c82739..7ed04974dd0fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Add support for Warm Indices Write Block on Flood Watermark breach ([#18375](https://github.com/opensearch-project/OpenSearch/pull/18375)) - Add support for custom index name resolver from cluster plugin ([#18593](https://github.com/opensearch-project/OpenSearch/pull/18593)) +- Rename WorkloadGroupTestUtil to WorkloadManagementTestUtil ([#18709](https://github.com/opensearch-project/OpenSearch/pull/18709)) - Disallow resize for Warm Index, add Parameterized ITs for close in remote store ([#18686](https://github.com/opensearch-project/OpenSearch/pull/18686)) - Ability to run Code Coverage with Gradle and produce the jacoco reports locally ([#18509](https://github.com/opensearch-project/OpenSearch/issues/18509)) - [Workload Management] Update logging and Javadoc, rename QueryGroup to WorkloadGroup ([#18711](https://github.com/opensearch-project/OpenSearch/issues/18711)) diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadGroupTestUtils.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementTestUtils.java similarity index 99% rename from plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadGroupTestUtils.java rename to plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementTestUtils.java index bac644a172c1e..bfc9ca356c112 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadGroupTestUtils.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementTestUtils.java @@ -38,7 +38,7 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; -public class WorkloadGroupTestUtils { +public class WorkloadManagementTestUtils { public static final String NAME_ONE = "workload_group_one"; public static final String NAME_TWO = "workload_group_two"; public static final String _ID_ONE = "AgfUO5Ja9yfsYlONlYi3TQ=="; diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateWorkloadGroupRequestTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateWorkloadGroupRequestTests.java index 31d3ea00b7bda..a0addd1e7b412 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateWorkloadGroupRequestTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateWorkloadGroupRequestTests.java @@ -17,8 +17,8 @@ import java.util.ArrayList; import java.util.List; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.assertEqualWorkloadGroups; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.workloadGroupOne; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.assertEqualWorkloadGroups; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.workloadGroupOne; public class CreateWorkloadGroupRequestTests extends OpenSearchTestCase { diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateWorkloadGroupResponseTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateWorkloadGroupResponseTests.java index d25050341f997..c741daf458268 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateWorkloadGroupResponseTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateWorkloadGroupResponseTests.java @@ -15,7 +15,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.plugin.wlm.WorkloadGroupTestUtils; +import org.opensearch.plugin.wlm.WorkloadManagementTestUtils; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -30,7 +30,7 @@ public class CreateWorkloadGroupResponseTests extends OpenSearchTestCase { * Test case to verify serialization and deserialization of CreateWorkloadGroupResponse. */ public void testSerialization() throws IOException { - CreateWorkloadGroupResponse response = new CreateWorkloadGroupResponse(WorkloadGroupTestUtils.workloadGroupOne, RestStatus.OK); + CreateWorkloadGroupResponse response = new CreateWorkloadGroupResponse(WorkloadManagementTestUtils.workloadGroupOne, RestStatus.OK); BytesStreamOutput out = new BytesStreamOutput(); response.writeTo(out); StreamInput streamInput = out.bytes().streamInput(); @@ -42,7 +42,7 @@ public void testSerialization() throws IOException { List listTwo = new ArrayList<>(); listOne.add(responseGroup); listTwo.add(otherResponseGroup); - WorkloadGroupTestUtils.assertEqualWorkloadGroups(listOne, listTwo, false); + WorkloadManagementTestUtils.assertEqualWorkloadGroups(listOne, listTwo, false); } /** @@ -50,7 +50,7 @@ public void testSerialization() throws IOException { */ public void testToXContentCreateWorkloadGroup() throws IOException { XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); - CreateWorkloadGroupResponse response = new CreateWorkloadGroupResponse(WorkloadGroupTestUtils.workloadGroupOne, RestStatus.OK); + CreateWorkloadGroupResponse response = new CreateWorkloadGroupResponse(WorkloadManagementTestUtils.workloadGroupOne, RestStatus.OK); String actual = response.toXContent(builder, mock(ToXContent.Params.class)).toString(); String expected = "{\n" + " \"_id\" : \"AgfUO5Ja9yfsYlONlYi3TQ==\",\n" diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/DeleteWorkloadGroupRequestTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/DeleteWorkloadGroupRequestTests.java index a7fa0939583c5..0ab32e537d5c5 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/DeleteWorkloadGroupRequestTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/DeleteWorkloadGroupRequestTests.java @@ -11,7 +11,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.plugin.wlm.WorkloadGroupTestUtils; +import org.opensearch.plugin.wlm.WorkloadManagementTestUtils; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -22,8 +22,8 @@ public class DeleteWorkloadGroupRequestTests extends OpenSearchTestCase { * Test case to verify the serialization and deserialization of DeleteWorkloadGroupRequest. */ public void testSerialization() throws IOException { - DeleteWorkloadGroupRequest request = new DeleteWorkloadGroupRequest(WorkloadGroupTestUtils.NAME_ONE); - assertEquals(WorkloadGroupTestUtils.NAME_ONE, request.getName()); + DeleteWorkloadGroupRequest request = new DeleteWorkloadGroupRequest(WorkloadManagementTestUtils.NAME_ONE); + assertEquals(WorkloadManagementTestUtils.NAME_ONE, request.getName()); BytesStreamOutput out = new BytesStreamOutput(); request.writeTo(out); StreamInput streamInput = out.bytes().streamInput(); diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetWorkloadGroupRequestTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetWorkloadGroupRequestTests.java index 832761d5084bb..675b34228669c 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetWorkloadGroupRequestTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetWorkloadGroupRequestTests.java @@ -10,7 +10,7 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.plugin.wlm.WorkloadGroupTestUtils; +import org.opensearch.plugin.wlm.WorkloadManagementTestUtils; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -21,8 +21,8 @@ public class GetWorkloadGroupRequestTests extends OpenSearchTestCase { * Test case to verify the serialization and deserialization of GetWorkloadGroupRequest. */ public void testSerialization() throws IOException { - GetWorkloadGroupRequest request = new GetWorkloadGroupRequest(WorkloadGroupTestUtils.NAME_ONE); - assertEquals(WorkloadGroupTestUtils.NAME_ONE, request.getName()); + GetWorkloadGroupRequest request = new GetWorkloadGroupRequest(WorkloadManagementTestUtils.NAME_ONE); + assertEquals(WorkloadManagementTestUtils.NAME_ONE, request.getName()); BytesStreamOutput out = new BytesStreamOutput(); request.writeTo(out); StreamInput streamInput = out.bytes().streamInput(); diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetWorkloadGroupResponseTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetWorkloadGroupResponseTests.java index dc0aeabc7a033..ffc2fa1f2ca4f 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetWorkloadGroupResponseTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetWorkloadGroupResponseTests.java @@ -15,7 +15,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.plugin.wlm.WorkloadGroupTestUtils; +import org.opensearch.plugin.wlm.WorkloadManagementTestUtils; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -31,7 +31,7 @@ public class GetWorkloadGroupResponseTests extends OpenSearchTestCase { */ public void testSerializationSingleWorkloadGroup() throws IOException { List list = new ArrayList<>(); - list.add(WorkloadGroupTestUtils.workloadGroupOne); + list.add(WorkloadManagementTestUtils.workloadGroupOne); GetWorkloadGroupResponse response = new GetWorkloadGroupResponse(list, RestStatus.OK); assertEquals(response.getWorkloadGroups(), list); @@ -41,15 +41,15 @@ public void testSerializationSingleWorkloadGroup() throws IOException { GetWorkloadGroupResponse otherResponse = new GetWorkloadGroupResponse(streamInput); assertEquals(response.getRestStatus(), otherResponse.getRestStatus()); - WorkloadGroupTestUtils.assertEqualWorkloadGroups(response.getWorkloadGroups(), otherResponse.getWorkloadGroups(), false); + WorkloadManagementTestUtils.assertEqualWorkloadGroups(response.getWorkloadGroups(), otherResponse.getWorkloadGroups(), false); } /** * Test case to verify the serialization and deserialization of GetWorkloadGroupResponse when the result contains multiple WorkloadGroups. */ public void testSerializationMultipleWorkloadGroup() throws IOException { - GetWorkloadGroupResponse response = new GetWorkloadGroupResponse(WorkloadGroupTestUtils.workloadGroupList(), RestStatus.OK); - assertEquals(response.getWorkloadGroups(), WorkloadGroupTestUtils.workloadGroupList()); + GetWorkloadGroupResponse response = new GetWorkloadGroupResponse(WorkloadManagementTestUtils.workloadGroupList(), RestStatus.OK); + assertEquals(response.getWorkloadGroups(), WorkloadManagementTestUtils.workloadGroupList()); BytesStreamOutput out = new BytesStreamOutput(); response.writeTo(out); @@ -58,7 +58,7 @@ public void testSerializationMultipleWorkloadGroup() throws IOException { GetWorkloadGroupResponse otherResponse = new GetWorkloadGroupResponse(streamInput); assertEquals(response.getRestStatus(), otherResponse.getRestStatus()); assertEquals(2, otherResponse.getWorkloadGroups().size()); - WorkloadGroupTestUtils.assertEqualWorkloadGroups(response.getWorkloadGroups(), otherResponse.getWorkloadGroups(), false); + WorkloadManagementTestUtils.assertEqualWorkloadGroups(response.getWorkloadGroups(), otherResponse.getWorkloadGroups(), false); } /** @@ -83,7 +83,7 @@ public void testSerializationNull() throws IOException { */ public void testToXContentGetSingleWorkloadGroup() throws IOException { List workloadGroupList = new ArrayList<>(); - workloadGroupList.add(WorkloadGroupTestUtils.workloadGroupOne); + workloadGroupList.add(WorkloadManagementTestUtils.workloadGroupOne); XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); GetWorkloadGroupResponse response = new GetWorkloadGroupResponse(workloadGroupList, RestStatus.OK); String actual = response.toXContent(builder, mock(ToXContent.Params.class)).toString(); @@ -108,8 +108,8 @@ public void testToXContentGetSingleWorkloadGroup() throws IOException { */ public void testToXContentGetMultipleWorkloadGroup() throws IOException { List workloadGroupList = new ArrayList<>(); - workloadGroupList.add(WorkloadGroupTestUtils.workloadGroupOne); - workloadGroupList.add(WorkloadGroupTestUtils.workloadGroupTwo); + workloadGroupList.add(WorkloadManagementTestUtils.workloadGroupOne); + workloadGroupList.add(WorkloadManagementTestUtils.workloadGroupTwo); XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); GetWorkloadGroupResponse response = new GetWorkloadGroupResponse(workloadGroupList, RestStatus.OK); String actual = response.toXContent(builder, mock(ToXContent.Params.class)).toString(); diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/TransportGetWorkloadGroupActionTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/TransportGetWorkloadGroupActionTests.java index cf12d9f6408cf..311eda7728d25 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/TransportGetWorkloadGroupActionTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/TransportGetWorkloadGroupActionTests.java @@ -17,9 +17,9 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.NAME_NONE_EXISTED; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.NAME_ONE; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.clusterState; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.NAME_NONE_EXISTED; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.NAME_ONE; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.clusterState; import static org.mockito.Mockito.mock; public class TransportGetWorkloadGroupActionTests extends OpenSearchTestCase { diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateWorkloadGroupRequestTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateWorkloadGroupRequestTests.java index e8d883da5c6eb..e45a0b0b6cfff 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateWorkloadGroupRequestTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateWorkloadGroupRequestTests.java @@ -19,8 +19,8 @@ import java.util.HashMap; import java.util.Map; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.NAME_ONE; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.workloadGroupOne; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.NAME_ONE; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.workloadGroupOne; public class UpdateWorkloadGroupRequestTests extends OpenSearchTestCase { diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateWorkloadGroupResponseTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateWorkloadGroupResponseTests.java index 97b9b9029373f..44437ee10aa1e 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateWorkloadGroupResponseTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateWorkloadGroupResponseTests.java @@ -15,14 +15,14 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.plugin.wlm.WorkloadGroupTestUtils; +import org.opensearch.plugin.wlm.WorkloadManagementTestUtils; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.workloadGroupOne; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.workloadGroupOne; import static org.mockito.Mockito.mock; public class UpdateWorkloadGroupResponseTests extends OpenSearchTestCase { @@ -43,7 +43,7 @@ public void testSerialization() throws IOException { List list2 = new ArrayList<>(); list1.add(responseGroup); list2.add(otherResponseGroup); - WorkloadGroupTestUtils.assertEqualWorkloadGroups(list1, list2, false); + WorkloadManagementTestUtils.assertEqualWorkloadGroups(list1, list2, false); } /** diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestDeleteWorkloadGroupActionTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestDeleteWorkloadGroupActionTests.java index 8ce5c869f4481..199b55754d132 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestDeleteWorkloadGroupActionTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestDeleteWorkloadGroupActionTests.java @@ -25,7 +25,7 @@ import org.mockito.ArgumentCaptor; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.NAME_ONE; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.NAME_ONE; import static org.opensearch.rest.RestRequest.Method.DELETE; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/WorkloadGroupPersistenceServiceTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/WorkloadGroupPersistenceServiceTests.java index e6ea2fc8a2700..51911b2b67df8 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/WorkloadGroupPersistenceServiceTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/WorkloadGroupPersistenceServiceTests.java @@ -21,7 +21,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; -import org.opensearch.plugin.wlm.WorkloadGroupTestUtils; +import org.opensearch.plugin.wlm.WorkloadManagementTestUtils; import org.opensearch.plugin.wlm.action.CreateWorkloadGroupResponse; import org.opensearch.plugin.wlm.action.DeleteWorkloadGroupRequest; import org.opensearch.plugin.wlm.action.UpdateWorkloadGroupRequest; @@ -44,20 +44,20 @@ import org.mockito.ArgumentCaptor; import static org.opensearch.cluster.metadata.WorkloadGroup.builder; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.NAME_NONE_EXISTED; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.NAME_ONE; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.NAME_TWO; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils._ID_ONE; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils._ID_TWO; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.assertEqualWorkloadGroups; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.clusterSettings; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.clusterSettingsSet; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.clusterState; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.preparePersistenceServiceSetup; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.workloadGroupList; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.workloadGroupOne; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.workloadGroupPersistenceService; -import static org.opensearch.plugin.wlm.WorkloadGroupTestUtils.workloadGroupTwo; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.NAME_NONE_EXISTED; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.NAME_ONE; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.NAME_TWO; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils._ID_ONE; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils._ID_TWO; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.assertEqualWorkloadGroups; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.clusterSettings; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.clusterSettingsSet; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.clusterState; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.preparePersistenceServiceSetup; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.workloadGroupList; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.workloadGroupOne; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.workloadGroupPersistenceService; +import static org.opensearch.plugin.wlm.WorkloadManagementTestUtils.workloadGroupTwo; import static org.opensearch.plugin.wlm.action.WorkloadGroupActionTestUtils.updateWorkloadGroupRequest; import static org.opensearch.plugin.wlm.service.WorkloadGroupPersistenceService.SOURCE; import static org.opensearch.plugin.wlm.service.WorkloadGroupPersistenceService.WORKLOAD_GROUP_COUNT_SETTING_NAME; @@ -209,7 +209,7 @@ public void testPersistInClusterStateMetadata() { ActionListener listener = mock(ActionListener.class); WorkloadGroupPersistenceService workloadGroupPersistenceService = new WorkloadGroupPersistenceService( clusterService, - WorkloadGroupTestUtils.settings(), + WorkloadManagementTestUtils.settings(), clusterSettings() ); workloadGroupPersistenceService.persistInClusterStateMetadata(workloadGroupOne, listener); @@ -225,7 +225,7 @@ public void testPersistInClusterStateMetadataInner() { ActionListener listener = mock(ActionListener.class); WorkloadGroupPersistenceService workloadGroupPersistenceService = new WorkloadGroupPersistenceService( clusterService, - WorkloadGroupTestUtils.settings(), + WorkloadManagementTestUtils.settings(), clusterSettings() ); ArgumentCaptor captor = ArgumentCaptor.forClass(ClusterStateUpdateTask.class); @@ -252,7 +252,7 @@ public void testPersistInClusterStateMetadataFailure() { ActionListener listener = mock(ActionListener.class); WorkloadGroupPersistenceService workloadGroupPersistenceService = new WorkloadGroupPersistenceService( clusterService, - WorkloadGroupTestUtils.settings(), + WorkloadManagementTestUtils.settings(), clusterSettings() ); doAnswer(invocation -> { @@ -275,23 +275,23 @@ public void testGetSingleWorkloadGroup() { WorkloadGroup workloadGroup = groups.get(0); List listOne = new ArrayList<>(); List listTwo = new ArrayList<>(); - listOne.add(WorkloadGroupTestUtils.workloadGroupOne); + listOne.add(WorkloadManagementTestUtils.workloadGroupOne); listTwo.add(workloadGroup); - WorkloadGroupTestUtils.assertEqualWorkloadGroups(listOne, listTwo, false); + WorkloadManagementTestUtils.assertEqualWorkloadGroups(listOne, listTwo, false); } /** * Tests getting all WorkloadGroups */ public void testGetAllWorkloadGroups() { - assertEquals(2, WorkloadGroupTestUtils.clusterState().metadata().workloadGroups().size()); + assertEquals(2, WorkloadManagementTestUtils.clusterState().metadata().workloadGroups().size()); Collection groupsCollections = WorkloadGroupPersistenceService.getFromClusterStateMetadata(null, clusterState()); List res = new ArrayList<>(groupsCollections); assertEquals(2, res.size()); Set currentNAME = res.stream().map(WorkloadGroup::getName).collect(Collectors.toSet()); - assertTrue(currentNAME.contains(WorkloadGroupTestUtils.NAME_ONE)); - assertTrue(currentNAME.contains(WorkloadGroupTestUtils.NAME_TWO)); - WorkloadGroupTestUtils.assertEqualWorkloadGroups(WorkloadGroupTestUtils.workloadGroupList(), res, false); + assertTrue(currentNAME.contains(WorkloadManagementTestUtils.NAME_ONE)); + assertTrue(currentNAME.contains(WorkloadManagementTestUtils.NAME_TWO)); + WorkloadManagementTestUtils.assertEqualWorkloadGroups(WorkloadManagementTestUtils.workloadGroupList(), res, false); } /** @@ -312,9 +312,9 @@ public void testGetNonExistedWorkloadGroups() { public void testMaxWorkloadGroupCount() { assertThrows( IllegalArgumentException.class, - () -> WorkloadGroupTestUtils.workloadGroupPersistenceService().setMaxWorkloadGroupCount(-1) + () -> WorkloadManagementTestUtils.workloadGroupPersistenceService().setMaxWorkloadGroupCount(-1) ); - WorkloadGroupPersistenceService workloadGroupPersistenceService = WorkloadGroupTestUtils.workloadGroupPersistenceService(); + WorkloadGroupPersistenceService workloadGroupPersistenceService = WorkloadManagementTestUtils.workloadGroupPersistenceService(); workloadGroupPersistenceService.setMaxWorkloadGroupCount(50); assertEquals(50, workloadGroupPersistenceService.getMaxWorkloadGroupCount()); } @@ -353,7 +353,7 @@ public void testDeleteInClusterStateMetadata() throws Exception { ActionListener listener = mock(ActionListener.class); WorkloadGroupPersistenceService workloadGroupPersistenceService = new WorkloadGroupPersistenceService( clusterService, - WorkloadGroupTestUtils.settings(), + WorkloadManagementTestUtils.settings(), clusterSettings() ); doAnswer(invocation -> { @@ -467,7 +467,7 @@ public void testUpdateInClusterStateMetadata() { ActionListener listener = mock(ActionListener.class); WorkloadGroupPersistenceService workloadGroupPersistenceService = new WorkloadGroupPersistenceService( clusterService, - WorkloadGroupTestUtils.settings(), + WorkloadManagementTestUtils.settings(), clusterSettings() ); workloadGroupPersistenceService.updateInClusterStateMetadata(null, listener); @@ -483,7 +483,7 @@ public void testUpdateInClusterStateMetadataInner() { ActionListener listener = mock(ActionListener.class); WorkloadGroupPersistenceService workloadGroupPersistenceService = new WorkloadGroupPersistenceService( clusterService, - WorkloadGroupTestUtils.settings(), + WorkloadManagementTestUtils.settings(), clusterSettings() ); UpdateWorkloadGroupRequest updateWorkloadGroupRequest = updateWorkloadGroupRequest( @@ -514,7 +514,7 @@ public void testUpdateInClusterStateMetadataFailure() { ActionListener listener = mock(ActionListener.class); WorkloadGroupPersistenceService workloadGroupPersistenceService = new WorkloadGroupPersistenceService( clusterService, - WorkloadGroupTestUtils.settings(), + WorkloadManagementTestUtils.settings(), clusterSettings() ); UpdateWorkloadGroupRequest updateWorkloadGroupRequest = updateWorkloadGroupRequest( From caa47f7c6d744cbb3a485ff79760f6985b476de8 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 9 Jul 2025 09:51:55 -0400 Subject: [PATCH 30/51] Add Prudhvi Godithi (GH: prudhvigodithi) as a maintainer (#18690) * Add Prudhvi Godithi (GH: prudhvigodithi) as a maintainer Signed-off-by: Craig Perkins * Keep spaces Signed-off-by: Craig Perkins * Fix spacing Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- MAINTAINERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index d00dc12621d98..6b18639282efa 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -28,6 +28,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Owais Kazi | [owaiskazi19](https://github.com/owaiskazi19) | Amazon | | Pan Guixin | [bugmakerrrrrr](https://github.com/bugmakerrrrrr) | ByteDance | | Peter Nied | [peternied](https://github.com/peternied) | Amazon | +| Prudhvi Godithi | [prudhvigodithi](https://github.com/prudhvigodithi) | Amazon | | Rishabh Maurya | [rishabhmaurya](https://github.com/rishabhmaurya) | Amazon | | Rishikesh Pasham | [Rishikesh1159](https://github.com/Rishikesh1159) | Amazon | | Sachin Kale | [sachinpkale](https://github.com/sachinpkale) | Amazon | From e1fc79e656e8f9f7d4d931d937eaceffdb18919a Mon Sep 17 00:00:00 2001 From: Bo Zhang Date: Wed, 9 Jul 2025 09:25:15 -0700 Subject: [PATCH 31/51] Pass index settings to system ingest processor factories. (#18708) Signed-off-by: Bo Zhang --- CHANGELOG.md | 1 + .../ExampleSystemIngestProcessorFactory.java | 25 +++++++- .../ExampleSystemIngestProcessorPlugin.java | 17 ++++++ ...mpleSystemIngestProcessorFactoryTests.java | 37 +++++++++++ ...ampleSystemIngestProcessorPluginTests.java | 6 ++ .../20_system_ingest_processor.yml | 61 ++++++++++++++++++- .../org/opensearch/ingest/IngestService.java | 23 ++++++- .../org/opensearch/plugins/IngestPlugin.java | 12 ++++ 8 files changed, 177 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ed04974dd0fe..07db5a30b9eaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add support for non-timing info in profiler ([#18460](https://github.com/opensearch-project/OpenSearch/issues/18460)) - Extend Approximation Framework to other numeric types ([#18530](https://github.com/opensearch-project/OpenSearch/issues/18530)) - Add Semantic Version field type mapper and extensive unit tests([#18454](https://github.com/opensearch-project/OpenSearch/pull/18454)) +- Pass index settings to system ingest processor factories. ([#18708](https://github.com/opensearch-project/OpenSearch/pull/18708)) ### Changed - Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570)) diff --git a/plugins/examples/system-ingest-processor/src/main/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorFactory.java b/plugins/examples/system-ingest-processor/src/main/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorFactory.java index d23079a113d1b..64a863e94c94f 100644 --- a/plugins/examples/system-ingest-processor/src/main/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorFactory.java +++ b/plugins/examples/system-ingest-processor/src/main/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorFactory.java @@ -8,15 +8,20 @@ package org.opensearch.example.systemingestprocessor; +import org.opensearch.common.settings.Settings; import org.opensearch.ingest.AbstractBatchingSystemProcessor; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.opensearch.example.systemingestprocessor.ExampleSystemIngestProcessorPlugin.TRIGGER_SETTING; import static org.opensearch.plugins.IngestPlugin.SystemIngestPipelineConfigKeys.INDEX_MAPPINGS; +import static org.opensearch.plugins.IngestPlugin.SystemIngestPipelineConfigKeys.INDEX_SETTINGS; import static org.opensearch.plugins.IngestPlugin.SystemIngestPipelineConfigKeys.INDEX_TEMPLATE_MAPPINGS; +import static org.opensearch.plugins.IngestPlugin.SystemIngestPipelineConfigKeys.INDEX_TEMPLATE_SETTINGS; /** * A factory to create the example system ingest processor @@ -55,17 +60,26 @@ protected ExampleSystemIngestProcessorFactory() { @Override protected AbstractBatchingSystemProcessor newProcessor(String tag, String description, Map config) { final List> mappings = new ArrayList<>(); + final List settings = new ArrayList<>(); final Object mappingFromIndex = config.get(INDEX_MAPPINGS); final Object mappingFromTemplates = config.get(INDEX_TEMPLATE_MAPPINGS); + final Object settingsFromIndex = config.get(INDEX_SETTINGS); + final Object settingsFromTemplates = config.get(INDEX_TEMPLATE_SETTINGS); if (mappingFromTemplates instanceof List) { mappings.addAll((List>) mappingFromTemplates); } if (mappingFromIndex instanceof Map) { mappings.add((Map) mappingFromIndex); } + if (settingsFromTemplates instanceof List) { + settings.addAll((Collection) settingsFromTemplates); + } + if (settingsFromIndex instanceof Settings) { + settings.add((Settings) settingsFromIndex); + } // If no config we are not able to create a processor so simply return a null to show no processor created - if (mappings.isEmpty()) { + if (mappings.isEmpty() && settings.isEmpty()) { return null; } @@ -87,6 +101,15 @@ protected AbstractBatchingSystemProcessor newProcessor(String tag, String descri } } + // If the trigger setting is configured then use it directly. + // When we rely on the v1 template to create the index there can be multiple settings and the later one can + // override the previous one so we need to loop through all the settings. + for (final Settings setting : settings) { + if (setting.hasValue(TRIGGER_SETTING.getKey())) { + isTriggerFieldFound = TRIGGER_SETTING.get(setting); + } + } + return isTriggerFieldFound ? new ExampleSystemIngestProcessor(tag, description, DEFAULT_BATCH_SIZE) : null; } diff --git a/plugins/examples/system-ingest-processor/src/main/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorPlugin.java b/plugins/examples/system-ingest-processor/src/main/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorPlugin.java index b1b5286b44b52..e370cbf8ae3e7 100644 --- a/plugins/examples/system-ingest-processor/src/main/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorPlugin.java +++ b/plugins/examples/system-ingest-processor/src/main/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorPlugin.java @@ -8,10 +8,12 @@ package org.opensearch.example.systemingestprocessor; +import org.opensearch.common.settings.Setting; import org.opensearch.ingest.Processor; import org.opensearch.plugins.IngestPlugin; import org.opensearch.plugins.Plugin; +import java.util.List; import java.util.Map; /** @@ -23,8 +25,23 @@ public class ExampleSystemIngestProcessorPlugin extends Plugin implements Ingest */ public ExampleSystemIngestProcessorPlugin() {} + /** + * A custom index setting which is used to control if we should create the example system ingest processor. + */ + public static final Setting TRIGGER_SETTING = Setting.boolSetting( + "index.example_system_ingest_processor_plugin.trigger_setting", + false, + Setting.Property.IndexScope, + Setting.Property.Dynamic + ); + @Override public Map getSystemIngestProcessors(Processor.Parameters parameters) { return Map.of(ExampleSystemIngestProcessorFactory.TYPE, new ExampleSystemIngestProcessorFactory()); } + + @Override + public List> getSettings() { + return List.of(TRIGGER_SETTING); + } } diff --git a/plugins/examples/system-ingest-processor/src/test/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorFactoryTests.java b/plugins/examples/system-ingest-processor/src/test/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorFactoryTests.java index 2c46c12b1d9bf..54b998f261a8a 100644 --- a/plugins/examples/system-ingest-processor/src/test/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorFactoryTests.java +++ b/plugins/examples/system-ingest-processor/src/test/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorFactoryTests.java @@ -8,6 +8,7 @@ package org.opensearch.example.systemingestprocessor; +import org.opensearch.common.settings.Settings; import org.opensearch.ingest.AbstractBatchingSystemProcessor; import org.opensearch.test.OpenSearchTestCase; @@ -18,8 +19,11 @@ import static org.opensearch.example.systemingestprocessor.ExampleSystemIngestProcessorFactory.DOC; import static org.opensearch.example.systemingestprocessor.ExampleSystemIngestProcessorFactory.PROPERTIES; import static org.opensearch.example.systemingestprocessor.ExampleSystemIngestProcessorFactory.TRIGGER_FIELD_NAME; +import static org.opensearch.example.systemingestprocessor.ExampleSystemIngestProcessorPlugin.TRIGGER_SETTING; import static org.opensearch.plugins.IngestPlugin.SystemIngestPipelineConfigKeys.INDEX_MAPPINGS; +import static org.opensearch.plugins.IngestPlugin.SystemIngestPipelineConfigKeys.INDEX_SETTINGS; import static org.opensearch.plugins.IngestPlugin.SystemIngestPipelineConfigKeys.INDEX_TEMPLATE_MAPPINGS; +import static org.opensearch.plugins.IngestPlugin.SystemIngestPipelineConfigKeys.INDEX_TEMPLATE_SETTINGS; public class ExampleSystemIngestProcessorFactoryTests extends OpenSearchTestCase { public void testNewProcessor_whenWithTriggerField_thenReturnProcessor() { @@ -55,4 +59,37 @@ public void testNewProcessor_whenNoMapping_thenReturnNull() { assertNull("Should not create an example system ingest processor when the mapping is not found.", processor); } + + public void testNewProcessor_whenWithTriggerSettingFromIndex_thenReturnProcessor() { + final ExampleSystemIngestProcessorFactory factory = new ExampleSystemIngestProcessorFactory(); + Settings triggerEnabled = Settings.builder().put(TRIGGER_SETTING.getKey(), true).build(); + + AbstractBatchingSystemProcessor processor = factory.newProcessor("tag", "description", Map.of(INDEX_SETTINGS, triggerEnabled)); + + assertNotNull("Should create an example system ingest processor when the trigger_setting is true.", processor); + assertTrue(processor instanceof ExampleSystemIngestProcessor); + } + + public void testNewProcessor_whenWithTriggerSettingFromTemplate_thenReturnProcessor() { + final ExampleSystemIngestProcessorFactory factory = new ExampleSystemIngestProcessorFactory(); + Settings triggerEnabled = Settings.builder().put(TRIGGER_SETTING.getKey(), true).build(); + + AbstractBatchingSystemProcessor processor = factory.newProcessor( + "tag", + "description", + Map.of(INDEX_TEMPLATE_SETTINGS, List.of(triggerEnabled)) + ); + + assertNotNull("Should create an example system ingest processor when the trigger_setting is true.", processor); + assertTrue(processor instanceof ExampleSystemIngestProcessor); + } + + public void testNewProcessor_whenWithTriggerSettingDisabled_thenReturnProcessor() { + final ExampleSystemIngestProcessorFactory factory = new ExampleSystemIngestProcessorFactory(); + Settings triggerDisabled = Settings.builder().put(TRIGGER_SETTING.getKey(), false).build(); + + AbstractBatchingSystemProcessor processor = factory.newProcessor("tag", "description", Map.of(INDEX_SETTINGS, triggerDisabled)); + + assertNull("Should not create an example system ingest processor when the trigger_setting is false.", processor); + } } diff --git a/plugins/examples/system-ingest-processor/src/test/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorPluginTests.java b/plugins/examples/system-ingest-processor/src/test/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorPluginTests.java index afad3a5e9366b..207fb024ea9c8 100644 --- a/plugins/examples/system-ingest-processor/src/test/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorPluginTests.java +++ b/plugins/examples/system-ingest-processor/src/test/java/org/opensearch/example/systemingestprocessor/ExampleSystemIngestProcessorPluginTests.java @@ -11,8 +11,10 @@ import org.opensearch.ingest.Processor; import org.opensearch.test.OpenSearchTestCase; +import java.util.List; import java.util.Map; +import static org.opensearch.example.systemingestprocessor.ExampleSystemIngestProcessorPlugin.TRIGGER_SETTING; import static org.mockito.Mockito.mock; public class ExampleSystemIngestProcessorPluginTests extends OpenSearchTestCase { @@ -27,4 +29,8 @@ public void testGetSystemIngestProcessors() { factories.get(ExampleSystemIngestProcessorFactory.TYPE) instanceof ExampleSystemIngestProcessorFactory ); } + + public void testGetSettings() { + assertEquals(List.of(TRIGGER_SETTING), plugin.getSettings()); + } } diff --git a/plugins/examples/system-ingest-processor/src/yamlRestTest/resources/rest-api-spec/test/example-system-ingest-processor/20_system_ingest_processor.yml b/plugins/examples/system-ingest-processor/src/yamlRestTest/resources/rest-api-spec/test/example-system-ingest-processor/20_system_ingest_processor.yml index 746e7b451c266..52ab654f03115 100644 --- a/plugins/examples/system-ingest-processor/src/yamlRestTest/resources/rest-api-spec/test/example-system-ingest-processor/20_system_ingest_processor.yml +++ b/plugins/examples/system-ingest-processor/src/yamlRestTest/resources/rest-api-spec/test/example-system-ingest-processor/20_system_ingest_processor.yml @@ -29,6 +29,28 @@ teardown: - match: _source.field_auto_added_by_system_ingest_processor: "This field is auto added by the example system ingest processor." +--- +"Processor injects a field on indexing a doc to existing index when trigger_setting is true": + - do: + indices.create: + index: test-index + body: + settings: + index.example_system_ingest_processor_plugin.trigger_setting: true + - do: + index: + index: test-index + id: 1 + body: + system_ingest_processor_trigger_field: "dummy value" + refresh: true + - do: + get: + index: test-index + id: 1 + - match: + _source.field_auto_added_by_system_ingest_processor: "This field is auto added by the example system ingest processor." + --- "Processor should not inject a field on indexing a doc to existing index when trigger field is not defined in the index mapping": - do: @@ -56,6 +78,7 @@ teardown: - skip: features: allowed_warnings - do: + # test v1 template indices.put_template: name: example-template body: @@ -79,7 +102,41 @@ teardown: id: 1 - match: _source.field_auto_added_by_system_ingest_processor: "This field is auto added by the example system ingest processor." + - do: + indices.delete_template: + name: example-template +--- +"Processor injects field when index is created from matching template where trigger_setting is true": + - skip: + features: allowed_warnings + - do: + # test v2 template + indices.put_index_template: + name: example-template + body: + index_patterns: ["template-*"] + template: + settings: + index.example_system_ingest_processor_plugin.trigger_setting: true + - do: + allowed_warnings: + - "index [template-index-1] matches multiple legacy templates [example-template, global], composable templates will only match a single template" + index: + index: template-index-1 + id: 1 + body: + system_ingest_processor_trigger_field: "dummy value" + refresh: true + - do: + get: + index: template-index-1 + id: 1 + - match: + _source.field_auto_added_by_system_ingest_processor: "This field is auto added by the example system ingest processor." + - do: + indices.delete_index_template: + name: example-template --- "Processor injects field on bulk indexing to existing index": - do: @@ -156,7 +213,9 @@ teardown: id: 2 - match: _source.field_auto_added_by_system_ingest_processor: "This field is auto added by the example system ingest processor." - + - do: + indices.delete_template: + name: bulk-template --- "Processor injects field on bulk update, upsert on existing/new documents": # Temporarily disable system ingest pipelines to insert without triggering the system ingest field diff --git a/server/src/main/java/org/opensearch/ingest/IngestService.java b/server/src/main/java/org/opensearch/ingest/IngestService.java index b340a0b81941d..50793d6e237db 100644 --- a/server/src/main/java/org/opensearch/ingest/IngestService.java +++ b/server/src/main/java/org/opensearch/ingest/IngestService.java @@ -107,7 +107,9 @@ import static org.opensearch.cluster.service.ClusterManagerTask.DELETE_PIPELINE; import static org.opensearch.cluster.service.ClusterManagerTask.PUT_PIPELINE; import static org.opensearch.plugins.IngestPlugin.SystemIngestPipelineConfigKeys.INDEX_MAPPINGS; +import static org.opensearch.plugins.IngestPlugin.SystemIngestPipelineConfigKeys.INDEX_SETTINGS; import static org.opensearch.plugins.IngestPlugin.SystemIngestPipelineConfigKeys.INDEX_TEMPLATE_MAPPINGS; +import static org.opensearch.plugins.IngestPlugin.SystemIngestPipelineConfigKeys.INDEX_TEMPLATE_SETTINGS; /** * Holder class for several ingest related services. @@ -272,7 +274,8 @@ public boolean resolveSystemIngestPipeline( // precedence), or if a V2 template does not match, any V1 templates String v2Template = MetadataIndexTemplateService.findV2Template(metadata, indexRequest.index(), false); if (v2Template != null) { - systemIngestPipelineId = getSystemIngestPipelineForTemplateV2(v2Template, indexRequest); + Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template); + systemIngestPipelineId = getSystemIngestPipelineForTemplateV2(v2Template, indexRequest, settings); } else { List templates = MetadataIndexTemplateService.findV1Templates( metadata, @@ -344,7 +347,7 @@ public boolean resolvePipelines(final DocWriteRequest originalRequest, final } if (this.isSystemIngestPipelineEnabled) { - systemIngestPipelineId = getSystemIngestPipelineForTemplateV2(v2Template, indexRequest); + systemIngestPipelineId = getSystemIngestPipelineForTemplateV2(v2Template, indexRequest, settings); } } else { List templates = MetadataIndexTemplateService.findV1Templates( @@ -433,9 +436,13 @@ private String getSystemIngestPipelineForTemplateV1( final String indexId = createIndexIdWithTemplateSuffix(indexRequest.index()); Pipeline ingestPipeline = systemIngestPipelineCache.getSystemIngestPipeline(indexId); if (ingestPipeline == null) { + final List settingsList = new ArrayList<>(); final List> mappingsMap = new ArrayList<>(); final Map pipelineConfig = new HashMap<>(); for (final IndexTemplateMetadata template : templates) { + if (template.settings() != null) { + settingsList.add(template.settings()); + } if (template.mappings() != null) { try { mappingsMap.add(MapperService.parseMapping(xContentRegistry, template.mappings().string())); @@ -453,6 +460,7 @@ private String getSystemIngestPipelineForTemplateV1( } pipelineConfig.put(INDEX_TEMPLATE_MAPPINGS, mappingsMap); + pipelineConfig.put(INDEX_TEMPLATE_SETTINGS, settingsList); ingestPipeline = createSystemIngestPipeline(indexId, pipelineConfig); } @@ -461,7 +469,11 @@ private String getSystemIngestPipelineForTemplateV1( return ingestPipeline.getProcessors().isEmpty() ? null : indexId; } - private String getSystemIngestPipelineForTemplateV2(@NonNull final String templateName, @NonNull final IndexRequest indexRequest) { + private String getSystemIngestPipelineForTemplateV2( + @NonNull final String templateName, + @NonNull final IndexRequest indexRequest, + final Settings settings + ) { // Here we cache it with index name + template as the suffix since currently we don't have the uuid. // We need to cache it so that later during execution we can find it by indexId to reuse it. final String indexId = createIndexIdWithTemplateSuffix(indexRequest.index()); @@ -491,6 +503,7 @@ private String getSystemIngestPipelineForTemplateV2(@NonNull final String templa } pipelineConfig.put(INDEX_TEMPLATE_MAPPINGS, mappingsMap); + pipelineConfig.put(INDEX_TEMPLATE_SETTINGS, settings == null ? Collections.emptyList() : List.of(settings)); ingestPipeline = createSystemIngestPipeline(indexId, pipelineConfig); } @@ -515,10 +528,14 @@ private String getSystemIngestPipelineForExistingIndex(@NonNull final IndexMetad if (ingestPipeline == null) { // no cache we will try to resolve the ingest pipeline based on the index configuration final MappingMetadata mappingMetadata = indexMetadata.mapping(); + final Settings settings = indexMetadata.getSettings(); final Map pipelineConfig = new HashMap<>(); if (mappingMetadata != null) { pipelineConfig.put(INDEX_MAPPINGS, mappingMetadata.getSourceAsMap()); } + if (settings != null) { + pipelineConfig.put(INDEX_SETTINGS, settings); + } ingestPipeline = createSystemIngestPipeline(indexId, pipelineConfig); } // we can get an empty pipeline from the cache diff --git a/server/src/main/java/org/opensearch/plugins/IngestPlugin.java b/server/src/main/java/org/opensearch/plugins/IngestPlugin.java index b169d354ff977..18709994fb1fb 100644 --- a/server/src/main/java/org/opensearch/plugins/IngestPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/IngestPlugin.java @@ -109,5 +109,17 @@ class SystemIngestPipelineConfigKeys { * ] */ public static final String INDEX_TEMPLATE_MAPPINGS = "index_template_mappings"; + + /** + * Use this key to access the settings{@link org.opensearch.common.settings.Settings} of the index from the config. + */ + public static final String INDEX_SETTINGS = "index_settings"; + + /** + * Use this key to access the settings{@link org.opensearch.common.settings.Settings} of the matched templates + * of the index from the config. If there are multiple matched templates the later one can override the setting of the previous one if merge + * rules are allowed. So this will be a list of settings. + */ + public static final String INDEX_TEMPLATE_SETTINGS = "index_template_settings"; } } From c2f41cf1fa68b618c2bba970b610a277abec068d Mon Sep 17 00:00:00 2001 From: Girish Jeyakumar <9464353+girishjeyakumar@users.noreply.github.com> Date: Wed, 9 Jul 2025 12:58:54 -0700 Subject: [PATCH 32/51] Include named queries from rescore contexts in matched_queries array (#18697) * Fix: Include named queries from rescore contexts in matched_queries array - Modified QueryRescoreContext to store ParsedQuery instead of just Query - Updated QueryRescorerBuilder to use context.toQuery() for capturing named queries - Enhanced MatchedQueriesPhase to collect named queries from all rescore contexts - Added comprehensive tests for both unit and REST API scenarios - Resolves inconsistency with Elasticsearch behavior where rescore named queries were not surfaced - Cleaned up unnecessary comments for better code readability Signed-off-by: Girish Jeyakumar * Address maintainer feedback: update RescoreContext interface design - Added parsedQuery() method to RescoreContext interface - Added getParsedQueries() method to RescoreContext interface - Updated QueryRescoreContext to override interface methods - Removed instanceof check in MatchedQueriesPhase by using interface method - Simplified assignment in QueryRescorerBuilder to one line Signed-off-by: Girish Jeyakumar * Complete interface replacement: replace query() with parsedQuery() and getQueries() with getParsedQueries() - Removed query() method from QueryRescoreContext - Removed getQueries() method from RescoreContext interface - Updated all callers to use parsedQuery().query() instead of query() - Updated DfsPhase to use getParsedQueries() instead of getQueries() - Updated QueryRescorer to use parsedQuery().query() for rescoring and explanations - Updated QueryRescorerBuilderTests to use parsedQuery().query() - This addresses the maintainer's feedback for a cleaner interface design Signed-off-by: Girish Jeyakumar * Finalize interface design and move changelog entry to Added section - Removed parsedQuery() method from RescoreContext base interface - Renamed getParsedQuery() to parsedQuery() in QueryRescoreContext for consistency - Updated all callers to use parsedQuery() method - Moved changelog entry to 'Added' section (more appropriate than 'Fixed') - Added proper imports for ParsedQuery to improve code readability - All tests pass and code formatting is correct Final interface design: - RescoreContext: only getParsedQueries() (returns List) - QueryRescoreContext: parsedQuery() + getParsedQueries() override - MatchedQueriesPhase: uses getParsedQueries().forEach() for clean iteration This addresses all maintainer feedback from @mch2 and provides a clean, well-tested implementation that surfaces named queries from rescore contexts in the matched_queries array. Signed-off-by: Girish Jeyakumar * Maintain backward compatibility for RescoreContext public API - Restore getQueries() method for backward compatibility - Add getParsedQueries() method for new functionality Signed-off-by: Girish Jeyakumar * Fix mixed cluster test failure by updating skip version to 3.2.0 Signed-off-by: girish jeyakumar --------- Signed-off-by: Girish Jeyakumar Signed-off-by: girish jeyakumar Signed-off-by: Marc Handalian Co-authored-by: Girish Jeyakumar Co-authored-by: Marc Handalian --- CHANGELOG.md | 1 + .../test/search/350_matched_queries.yml | 98 +++++++++++++++++++ .../org/opensearch/search/dfs/DfsPhase.java | 6 +- .../fetch/subphase/MatchedQueriesPhase.java | 7 ++ .../search/rescore/QueryRescorer.java | 23 +++-- .../search/rescore/QueryRescorerBuilder.java | 5 +- .../search/rescore/RescoreContext.java | 8 ++ .../rescore/QueryRescorerBuilderTests.java | 46 ++++++++- 8 files changed, 180 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07db5a30b9eaa..ae527de80e952 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Extend Approximation Framework to other numeric types ([#18530](https://github.com/opensearch-project/OpenSearch/issues/18530)) - Add Semantic Version field type mapper and extensive unit tests([#18454](https://github.com/opensearch-project/OpenSearch/pull/18454)) - Pass index settings to system ingest processor factories. ([#18708](https://github.com/opensearch-project/OpenSearch/pull/18708)) +- Include named queries from rescore contexts in matched_queries array ([#18697](https://github.com/opensearch-project/OpenSearch/pull/18697)) ### Changed - Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570)) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/350_matched_queries.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/350_matched_queries.yml index 08a20df093c01..4094e2f08eb1a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/350_matched_queries.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/350_matched_queries.yml @@ -101,3 +101,101 @@ setup: - match: { hits.hits.0.matched_queries.match_field_2: 10 } - length: { hits.hits.1.matched_queries: 1 } - match: { hits.hits.1.matched_queries.match_field_1: 1 } + +--- + +"named queries in rescore": + - skip: + version: " - 3.1.99" + reason: "named queries in rescore is supported in 3.2.0 and above" + + - do: + indices.create: + index: test + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "test_1", "_id" : "1" } }' + - '{"field" : 1, "title": "hello world" }' + - '{ "index" : { "_index" : "test_1", "_id" : "2" } }' + - '{"field" : 2, "title": "hello universe" }' + + - do: + search: + index: test_1 + body: + query: + match: { + field: { + query: 1, + _name: main_query + } + } + rescore: + window_size: 10 + query: + rescore_query: + match: { + title: { + query: "hello", + _name: rescore_query + } + } + query_weight: 0.5 + rescore_query_weight: 1.5 + + - match: { hits.total.value: 1 } + - length: { hits.hits.0.matched_queries: 2 } + - match: { hits.hits.0.matched_queries: [ "main_query", "rescore_query" ] } + +--- + +"named queries in rescore with scores": + - skip: + version: " - 3.1.99" + reason: "named queries in rescore is supported in 3.2.0 and above" + + - do: + indices.create: + index: test + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "test_1", "_id" : "1" } }' + - '{"field" : 1, "title": "hello world" }' + - '{ "index" : { "_index" : "test_1", "_id" : "2" } }' + - '{"field" : 2, "title": "hello universe" }' + + - do: + search: + include_named_queries_score: true + index: test_1 + body: + query: + match: { + field: { + query: 1, + _name: main_query + } + } + rescore: + window_size: 10 + query: + rescore_query: + match: { + title: { + query: "hello", + _name: rescore_query + } + } + query_weight: 0.5 + rescore_query_weight: 1.5 + + - match: { hits.total.value: 1 } + - length: { hits.hits.0.matched_queries: 2 } + - gte: { hits.hits.0.matched_queries.main_query: 0.0 } + - gte: { hits.hits.0.matched_queries.rescore_query: 0.0 } diff --git a/server/src/main/java/org/opensearch/search/dfs/DfsPhase.java b/server/src/main/java/org/opensearch/search/dfs/DfsPhase.java index b5f6c082a18c5..fc4b1d3e4b828 100644 --- a/server/src/main/java/org/opensearch/search/dfs/DfsPhase.java +++ b/server/src/main/java/org/opensearch/search/dfs/DfsPhase.java @@ -35,10 +35,10 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.TermStatistics; import org.opensearch.core.tasks.TaskCancelledException; +import org.opensearch.index.query.ParsedQuery; import org.opensearch.search.internal.SearchContext; import org.opensearch.search.rescore.RescoreContext; @@ -86,8 +86,8 @@ public CollectionStatistics collectionStatistics(String field) throws IOExceptio searcher.createWeight(context.searcher().rewrite(context.query()), ScoreMode.COMPLETE, 1); for (RescoreContext rescoreContext : context.rescore()) { - for (Query query : rescoreContext.getQueries()) { - searcher.createWeight(context.searcher().rewrite(query), ScoreMode.COMPLETE, 1); + for (ParsedQuery parsedQuery : rescoreContext.getParsedQueries()) { + searcher.createWeight(context.searcher().rewrite(parsedQuery.query()), ScoreMode.COMPLETE, 1); } } diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/MatchedQueriesPhase.java b/server/src/main/java/org/opensearch/search/fetch/subphase/MatchedQueriesPhase.java index 406d9c8b4bc03..427946e7e2b21 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/MatchedQueriesPhase.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/MatchedQueriesPhase.java @@ -41,6 +41,7 @@ import org.opensearch.search.fetch.FetchContext; import org.opensearch.search.fetch.FetchSubPhase; import org.opensearch.search.fetch.FetchSubPhaseProcessor; +import org.opensearch.search.rescore.RescoreContext; import java.io.IOException; import java.util.ArrayList; @@ -65,6 +66,12 @@ public FetchSubPhaseProcessor getProcessor(FetchContext context) throws IOExcept if (context.parsedPostFilter() != null) { namedQueries.putAll(context.parsedPostFilter().namedFilters()); } + if (context.rescore() != null) { + for (RescoreContext rescoreContext : context.rescore()) { + rescoreContext.getParsedQueries().forEach(query -> namedQueries.putAll(query.namedFilters())); + } + } + if (namedQueries.isEmpty()) { return null; } diff --git a/server/src/main/java/org/opensearch/search/rescore/QueryRescorer.java b/server/src/main/java/org/opensearch/search/rescore/QueryRescorer.java index 008bff18cea5b..e95fbb292deb5 100644 --- a/server/src/main/java/org/opensearch/search/rescore/QueryRescorer.java +++ b/server/src/main/java/org/opensearch/search/rescore/QueryRescorer.java @@ -37,6 +37,8 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; +import org.opensearch.index.query.ParsedQuery; +import org.opensearch.search.rescore.QueryRescorer.QueryRescoreContext; import java.io.IOException; import java.util.Arrays; @@ -67,7 +69,7 @@ public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext r final QueryRescoreContext rescore = (QueryRescoreContext) rescoreContext; - org.apache.lucene.search.Rescorer rescorer = new org.apache.lucene.search.QueryRescorer(rescore.query()) { + org.apache.lucene.search.Rescorer rescorer = new org.apache.lucene.search.QueryRescorer(rescore.parsedQuery().query()) { @Override protected float combine(float firstPassScore, boolean secondPassMatches, float secondPassScore) { @@ -120,7 +122,7 @@ public Explanation explain(int topLevelDocId, IndexSearcher searcher, RescoreCon prim = Explanation.noMatch("First pass did not match", sourceExplanation); } if (rescoreContext.isRescored(topLevelDocId)) { - Explanation rescoreExplain = searcher.explain(rescore.query(), topLevelDocId); + Explanation rescoreExplain = searcher.explain(rescore.parsedQuery().query(), topLevelDocId); // NOTE: we don't use Lucene's Rescorer.explain because we want to insert our own description with which ScoreMode was used. // Maybe we should add QueryRescorer.explainCombine to Lucene? if (rescoreExplain != null && rescoreExplain.isMatch()) { @@ -190,7 +192,7 @@ private TopDocs combine(TopDocs in, TopDocs resorted, QueryRescoreContext ctx) { * @opensearch.internal */ public static class QueryRescoreContext extends RescoreContext { - private Query query; + private ParsedQuery parsedQuery; private float queryWeight = 1.0f; private float rescoreQueryWeight = 1.0f; private QueryRescoreMode scoreMode; @@ -200,17 +202,22 @@ public QueryRescoreContext(int windowSize) { this.scoreMode = QueryRescoreMode.Total; } - public void setQuery(Query query) { - this.query = query; + public void setParsedQuery(ParsedQuery parsedQuery) { + this.parsedQuery = parsedQuery; + } + + public ParsedQuery parsedQuery() { + return parsedQuery; } @Override public List getQueries() { - return Collections.singletonList(query); + return parsedQuery != null ? Collections.singletonList(parsedQuery.query()) : Collections.emptyList(); } - public Query query() { - return query; + @Override + public List getParsedQueries() { + return parsedQuery != null ? Collections.singletonList(parsedQuery) : Collections.emptyList(); } public float queryWeight() { diff --git a/server/src/main/java/org/opensearch/search/rescore/QueryRescorerBuilder.java b/server/src/main/java/org/opensearch/search/rescore/QueryRescorerBuilder.java index aea10755d4e42..19afb97dc9e79 100644 --- a/server/src/main/java/org/opensearch/search/rescore/QueryRescorerBuilder.java +++ b/server/src/main/java/org/opensearch/search/rescore/QueryRescorerBuilder.java @@ -190,8 +190,9 @@ public static QueryRescorerBuilder fromXContent(XContentParser parser) throws IO @Override public QueryRescoreContext innerBuildContext(int windowSize, QueryShardContext context) throws IOException { QueryRescoreContext queryRescoreContext = new QueryRescoreContext(windowSize); - // query is rewritten at this point already - queryRescoreContext.setQuery(queryBuilder.toQuery(context)); + + queryRescoreContext.setParsedQuery(context.toQuery(queryBuilder)); + queryRescoreContext.setQueryWeight(this.queryWeight); queryRescoreContext.setRescoreQueryWeight(this.rescoreQueryWeight); queryRescoreContext.setScoreMode(this.scoreMode); diff --git a/server/src/main/java/org/opensearch/search/rescore/RescoreContext.java b/server/src/main/java/org/opensearch/search/rescore/RescoreContext.java index d4b87f004b58f..771e12324aac4 100644 --- a/server/src/main/java/org/opensearch/search/rescore/RescoreContext.java +++ b/server/src/main/java/org/opensearch/search/rescore/RescoreContext.java @@ -34,6 +34,7 @@ import org.apache.lucene.search.Query; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.index.query.ParsedQuery; import java.util.Collections; import java.util.List; @@ -93,4 +94,11 @@ public Set getRescoredDocs() { public List getQueries() { return Collections.emptyList(); } + + /** + * Returns parsed queries associated with the rescorer + */ + public List getParsedQueries() { + return Collections.emptyList(); + } } diff --git a/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java index a71c18aa2266b..7324bac55c21f 100644 --- a/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java @@ -190,7 +190,7 @@ public MappedFieldType fieldMapper(String name) { : rescoreBuilder.windowSize().intValue(); assertEquals(expectedWindowSize, rescoreContext.getWindowSize()); Query expectedQuery = Rewriteable.rewrite(rescoreBuilder.getRescoreQuery(), mockShardContext).toQuery(mockShardContext); - assertEquals(expectedQuery, rescoreContext.query()); + assertEquals(expectedQuery, rescoreContext.parsedQuery().query()); assertEquals(rescoreBuilder.getQueryWeight(), rescoreContext.queryWeight(), Float.MIN_VALUE); assertEquals(rescoreBuilder.getRescoreQueryWeight(), rescoreContext.rescoreQueryWeight(), Float.MIN_VALUE); assertEquals(rescoreBuilder.getScoreMode(), rescoreContext.scoreMode()); @@ -202,6 +202,50 @@ public void testRescoreQueryNull() throws IOException { assertEquals("rescore_query cannot be null", e.getMessage()); } + /** + * Test that named queries from rescore contexts are captured + */ + public void testRescoreNamedQueries() throws IOException { + final long nowInMillis = randomNonNegativeLong(); + Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); + + QueryShardContext mockShardContext = new QueryShardContext( + 0, + idxSettings, + BigArrays.NON_RECYCLING_INSTANCE, + null, + null, + null, + null, + null, + xContentRegistry(), + namedWriteableRegistry, + null, + null, + () -> nowInMillis, + null, + null, + () -> true, + null + ) { + @Override + public MappedFieldType fieldMapper(String name) { + TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, createDefaultIndexAnalyzers()); + return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType(); + } + }; + + QueryBuilder namedQueryBuilder = new MatchAllQueryBuilder().queryName("test_rescore_query"); + QueryRescorerBuilder rescoreBuilder = new QueryRescorerBuilder(namedQueryBuilder); + QueryRescoreContext rescoreContext = (QueryRescoreContext) rescoreBuilder.buildContext(mockShardContext); + assertNotNull(rescoreContext.parsedQuery()); + assertNotNull(rescoreContext.parsedQuery().namedFilters()); + assertEquals(1, rescoreContext.parsedQuery().namedFilters().size()); + assertTrue(rescoreContext.parsedQuery().namedFilters().containsKey("test_rescore_query")); + assertNotNull(rescoreContext.parsedQuery().namedFilters().get("test_rescore_query")); + } + class AlwaysRewriteQueryBuilder extends MatchAllQueryBuilder { @Override From b609a50a3bf7d574b8e09a77f590fce1bcbf6a9c Mon Sep 17 00:00:00 2001 From: Finn Date: Wed, 9 Jul 2025 15:06:17 -0700 Subject: [PATCH 33/51] Move gRPC transitive dependencies to expected version (#18672) * Some explicit transient gRPC deps are not the expected version. grpc-netty-shaded:1.68.2 expects perfmark-api:0.27.0 guava:33.2.1-jre expects failureaccess:1.0.2 Signed-off-by: Finn Carroll * Update license shas. Signed-off-by: Finn Carroll * Changelog. Signed-off-by: Finn Carroll * Rebase changelog. Signed-off-by: Finn Carroll --------- Signed-off-by: Finn Carroll --- CHANGELOG.md | 2 ++ plugins/transport-grpc/build.gradle | 4 ++-- plugins/transport-grpc/licenses/failureaccess-1.0.1.jar.sha1 | 1 - plugins/transport-grpc/licenses/failureaccess-1.0.2.jar.sha1 | 1 + plugins/transport-grpc/licenses/perfmark-api-0.26.0.jar.sha1 | 1 - plugins/transport-grpc/licenses/perfmark-api-0.27.0.jar.sha1 | 1 + 6 files changed, 6 insertions(+), 4 deletions(-) delete mode 100644 plugins/transport-grpc/licenses/failureaccess-1.0.1.jar.sha1 create mode 100644 plugins/transport-grpc/licenses/failureaccess-1.0.2.jar.sha1 delete mode 100644 plugins/transport-grpc/licenses/perfmark-api-0.26.0.jar.sha1 create mode 100644 plugins/transport-grpc/licenses/perfmark-api-0.27.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index ae527de80e952..59e5428aa4b69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.squareup.okio:okio` from 3.13.0 to 3.15.0 ([#18645](https://github.com/opensearch-project/OpenSearch/pull/18645), [#18689](https://github.com/opensearch-project/OpenSearch/pull/18689)) - Bump `com.netflix.nebula.ospackage-base` from 11.11.2 to 12.0.0 ([#18646](https://github.com/opensearch-project/OpenSearch/pull/18646)) - Bump `com.azure:azure-storage-blob` from 12.30.0 to 12.30.1 ([#18644](https://github.com/opensearch-project/OpenSearch/pull/18644)) +- Bump `com.google.guava:failureaccess` from 1.0.1 to 1.0.2 ([#18672](https://github.com/opensearch-project/OpenSearch/pull/18672)) +- Bump `io.perfmark:perfmark-api` from 0.26.0 to 0.27.0 ([#18672](https://github.com/opensearch-project/OpenSearch/pull/18672)) - Bump `org.bouncycastle:bctls-fips` from 2.0.19 to 2.0.20 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) - Bump `org.bouncycastle:bcpkix-fips` from 2.0.7 to 2.0.8 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) - Bump `org.bouncycastle:bcpg-fips` from 2.0.10 to 2.0.11 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) diff --git a/plugins/transport-grpc/build.gradle b/plugins/transport-grpc/build.gradle index 2e5db8116ee63..5553fdca716ff 100644 --- a/plugins/transport-grpc/build.gradle +++ b/plugins/transport-grpc/build.gradle @@ -25,7 +25,7 @@ dependencies { compileOnly "com.google.code.findbugs:jsr305:3.0.2" runtimeOnly "com.google.guava:guava:${versions.guava}" implementation "com.google.errorprone:error_prone_annotations:2.24.1" - implementation "com.google.guava:failureaccess:1.0.1" + implementation "com.google.guava:failureaccess:1.0.2" implementation "io.grpc:grpc-api:${versions.grpc}" implementation "io.grpc:grpc-core:${versions.grpc}" implementation "io.grpc:grpc-netty-shaded:${versions.grpc}" @@ -34,7 +34,7 @@ dependencies { implementation "io.grpc:grpc-services:${versions.grpc}" implementation "io.grpc:grpc-stub:${versions.grpc}" implementation "io.grpc:grpc-util:${versions.grpc}" - implementation "io.perfmark:perfmark-api:0.26.0" + implementation "io.perfmark:perfmark-api:0.27.0" implementation "org.opensearch:protobufs:0.3.0" testImplementation project(':test:framework') } diff --git a/plugins/transport-grpc/licenses/failureaccess-1.0.1.jar.sha1 b/plugins/transport-grpc/licenses/failureaccess-1.0.1.jar.sha1 deleted file mode 100644 index 4798b37e20691..0000000000000 --- a/plugins/transport-grpc/licenses/failureaccess-1.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1dcf1de382a0bf95a3d8b0849546c88bac1292c9 \ No newline at end of file diff --git a/plugins/transport-grpc/licenses/failureaccess-1.0.2.jar.sha1 b/plugins/transport-grpc/licenses/failureaccess-1.0.2.jar.sha1 new file mode 100644 index 0000000000000..43cb5aa469900 --- /dev/null +++ b/plugins/transport-grpc/licenses/failureaccess-1.0.2.jar.sha1 @@ -0,0 +1 @@ +c4a06a64e650562f30b7bf9aaec1bfed43aca12b diff --git a/plugins/transport-grpc/licenses/perfmark-api-0.26.0.jar.sha1 b/plugins/transport-grpc/licenses/perfmark-api-0.26.0.jar.sha1 deleted file mode 100644 index abf1becd13298..0000000000000 --- a/plugins/transport-grpc/licenses/perfmark-api-0.26.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ef65452adaf20bf7d12ef55913aba24037b82738 \ No newline at end of file diff --git a/plugins/transport-grpc/licenses/perfmark-api-0.27.0.jar.sha1 b/plugins/transport-grpc/licenses/perfmark-api-0.27.0.jar.sha1 new file mode 100644 index 0000000000000..54651a33e5194 --- /dev/null +++ b/plugins/transport-grpc/licenses/perfmark-api-0.27.0.jar.sha1 @@ -0,0 +1 @@ +f86f575a41b091786a4b027cd9c0c1d2e3fc1c01 From 1821e96863920974666dbb86b7d56b865595208f Mon Sep 17 00:00:00 2001 From: Ruirui Zhang Date: Wed, 9 Jul 2025 16:51:36 -0700 Subject: [PATCH 34/51] Modify wlm logging message (#18712) Signed-off-by: Ruirui Zhang --- CHANGELOG.md | 1 + .../java/org/opensearch/wlm/WorkloadGroupService.java | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59e5428aa4b69..5f4ba31a2b630 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add last index request timestamp columns to the `_cat/indices` API. ([10766](https://github.com/opensearch-project/OpenSearch/issues/10766)) - Introduce a new pull-based ingestion plugin for file-based indexing (for local testing) ([#18591](https://github.com/opensearch-project/OpenSearch/pull/18591)) - Add support for search pipeline in search and msearch template ([#18564](https://github.com/opensearch-project/OpenSearch/pull/18564)) +- [Workload Management] Modify logging message in WorkloadGroupService ([#18712](https://github.com/opensearch-project/OpenSearch/pull/18712)) - Add BooleanQuery rewrite moving constant-scoring must clauses to filter clauses ([#18510](https://github.com/opensearch-project/OpenSearch/issues/18510)) - Add functionality for plugins to inject QueryCollectorContext during QueryPhase ([#18637](https://github.com/opensearch-project/OpenSearch/pull/18637)) - Add support for non-timing info in profiler ([#18460](https://github.com/opensearch-project/OpenSearch/issues/18460)) diff --git a/server/src/main/java/org/opensearch/wlm/WorkloadGroupService.java b/server/src/main/java/org/opensearch/wlm/WorkloadGroupService.java index 86ef0973762db..a36a5a8bbfd31 100644 --- a/server/src/main/java/org/opensearch/wlm/WorkloadGroupService.java +++ b/server/src/main/java/org/opensearch/wlm/WorkloadGroupService.java @@ -288,11 +288,15 @@ public void rejectIfNeeded(String workloadGroupId) { if (threshold < lastRecordedUsage) { reject = true; reason.append(resourceType) - .append(" limit is breaching for ENFORCED type WorkloadGroup: (") + .append(" limit is breaching for workload group ") + .append(workloadGroup.get_id()) + .append(", ") .append(threshold) .append(" < ") .append(lastRecordedUsage) - .append("). "); + .append(", wlm mode is ") + .append(workloadGroup.getResiliencyMode()) + .append(". "); workloadGroupState.getResourceState().get(resourceType).rejections.inc(); // should not double count even if both the resource limits are breaching break; From 5ac8177498f975959fcacdee7aa78bc6521c6587 Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Wed, 9 Jul 2025 20:20:21 -0700 Subject: [PATCH 35/51] [WLM] Introduce rule cardinality check (#18663) Signed-off-by: Kaushal Kumar --- CHANGELOG.md | 1 + .../org/opensearch/rule/RuleQueryMapper.java | 7 +++ .../IndexStoredRulePersistenceService.java | 45 ++++++++++++++++++- .../storage/IndexBasedRuleQueryMapper.java | 5 +++ ...ndexStoredRulePersistenceServiceTests.java | 28 ++++++++++++ .../plugin/wlm/WorkloadManagementPlugin.java | 8 +++- 6 files changed, 92 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f4ba31a2b630..ac002af112f37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add Semantic Version field type mapper and extensive unit tests([#18454](https://github.com/opensearch-project/OpenSearch/pull/18454)) - Pass index settings to system ingest processor factories. ([#18708](https://github.com/opensearch-project/OpenSearch/pull/18708)) - Include named queries from rescore contexts in matched_queries array ([#18697](https://github.com/opensearch-project/OpenSearch/pull/18697)) +- Add the configurable limit on rule cardinality ([#18663](https://github.com/opensearch-project/OpenSearch/pull/18663)) ### Changed - Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570)) diff --git a/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/RuleQueryMapper.java b/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/RuleQueryMapper.java index f0ee052790b4b..cb116b7626dd2 100644 --- a/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/RuleQueryMapper.java +++ b/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/RuleQueryMapper.java @@ -24,4 +24,11 @@ public interface RuleQueryMapper { * @return */ T from(GetRuleRequest request); + + /** + * This method returns the cardinality query for the rule, this query should + * be constructed in such a way that it can be used to calculate the cardinality of the rules + * @return + */ + T getCardinalityQuery(); } diff --git a/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/service/IndexStoredRulePersistenceService.java b/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/service/IndexStoredRulePersistenceService.java index d7a6f396bed74..73e68afd48f0e 100644 --- a/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/service/IndexStoredRulePersistenceService.java +++ b/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/service/IndexStoredRulePersistenceService.java @@ -19,9 +19,11 @@ import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.update.UpdateRequest; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Setting; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.index.engine.DocumentMissingException; import org.opensearch.index.query.QueryBuilder; @@ -53,8 +55,28 @@ */ public class IndexStoredRulePersistenceService implements RulePersistenceService { /** - * The system index name used for storing rules + * Default value for max rules count */ + public static final int DEFAULT_MAX_ALLOWED_RULE_COUNT = 200; + + /** + * max wlm rules setting name + */ + public static final String MAX_RULES_COUNT_SETTING_NAME = "wlm.autotagging.max_rules"; + + /** + * max wlm rules setting + */ + public static final Setting MAX_WLM_RULES_SETTING = Setting.intSetting( + MAX_RULES_COUNT_SETTING_NAME, + DEFAULT_MAX_ALLOWED_RULE_COUNT, + 10, + 500, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private int maxAllowedRulesCount; private final String indexName; private final Client client; private final ClusterService clusterService; @@ -87,6 +109,12 @@ public IndexStoredRulePersistenceService( this.maxRulesPerPage = maxRulesPerPage; this.parser = parser; this.queryBuilder = queryBuilder; + this.maxAllowedRulesCount = MAX_WLM_RULES_SETTING.get(clusterService.getSettings()); + clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_WLM_RULES_SETTING, this::setMaxAllowedRules); + } + + private void setMaxAllowedRules(int maxAllowedRules) { + this.maxAllowedRulesCount = maxAllowedRules; } /** @@ -101,12 +129,27 @@ public void createRule(CreateRuleRequest request, ActionListener persistRule(rule, listener), listener::onFailure)); } } } + private void performCardinalityCheck(ActionListener listener) { + SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(queryBuilder.getCardinalityQuery()).get(); + if (searchResponse.getHits().getTotalHits() != null && searchResponse.getHits().getTotalHits().value() >= maxAllowedRulesCount) { + listener.onFailure( + new OpenSearchRejectedExecutionException( + "This create operation will violate" + + " the cardinality limit of " + + DEFAULT_MAX_ALLOWED_RULE_COUNT + + ". Please delete some stale or redundant rules first" + ) + ); + } + } + /** * Validates that no existing rule has the same attribute map as the given rule. * This validation must be performed one at a time to prevent writing duplicate rules. diff --git a/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/storage/IndexBasedRuleQueryMapper.java b/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/storage/IndexBasedRuleQueryMapper.java index 11b0ad5e564fb..5189c4b2ce48b 100644 --- a/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/storage/IndexBasedRuleQueryMapper.java +++ b/modules/autotagging-commons/common/src/main/java/org/opensearch/rule/storage/IndexBasedRuleQueryMapper.java @@ -53,4 +53,9 @@ public QueryBuilder from(GetRuleRequest request) { } return boolQuery; } + + @Override + public QueryBuilder getCardinalityQuery() { + return QueryBuilders.matchAllQuery(); + } } diff --git a/modules/autotagging-commons/common/src/test/java/org/opensearch/rule/service/IndexStoredRulePersistenceServiceTests.java b/modules/autotagging-commons/common/src/test/java/org/opensearch/rule/service/IndexStoredRulePersistenceServiceTests.java index e348d104cfe27..5d90dd3d0b7be 100644 --- a/modules/autotagging-commons/common/src/test/java/org/opensearch/rule/service/IndexStoredRulePersistenceServiceTests.java +++ b/modules/autotagging-commons/common/src/test/java/org/opensearch/rule/service/IndexStoredRulePersistenceServiceTests.java @@ -21,10 +21,12 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.action.ActionFuture; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.engine.DocumentMissingException; import org.opensearch.index.query.QueryBuilder; @@ -50,6 +52,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -99,6 +102,11 @@ public void setUp() throws Exception { client = setUpMockClient(searchRequestBuilder); rule = mock(Rule.class); clusterService = mock(ClusterService.class); + Settings testSettings = Settings.EMPTY; + ClusterSettings clusterSettings = new ClusterSettings(testSettings, new HashSet<>()); + when(clusterService.getSettings()).thenReturn(testSettings); + clusterSettings.registerSetting(IndexStoredRulePersistenceService.MAX_WLM_RULES_SETTING); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); ClusterState clusterState = mock(ClusterState.class); Metadata metadata = mock(Metadata.class); when(clusterService.state()).thenReturn(clusterState); @@ -109,6 +117,7 @@ public void setUp() throws Exception { queryBuilder = mock(QueryBuilder.class); when(queryBuilder.filter(any())).thenReturn(queryBuilder); when(ruleQueryMapper.from(any(GetRuleRequest.class))).thenReturn(queryBuilder); + when(ruleQueryMapper.getCardinalityQuery()).thenReturn(mock(QueryBuilder.class)); when(ruleEntityParser.parse(anyString())).thenReturn(rule); rulePersistenceService = new IndexStoredRulePersistenceService( @@ -144,6 +153,25 @@ public void testCreateRuleOnExistingIndex() throws Exception { assertNotNull(responseCaptor.getValue().getRule()); } + public void testCardinalityCheckBasedFailure() throws Exception { + CreateRuleRequest createRuleRequest = mock(CreateRuleRequest.class); + when(createRuleRequest.getRule()).thenReturn(rule); + when(rule.toXContent(any(), any())).thenAnswer(invocation -> invocation.getArgument(0)); + + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.getHits()).thenReturn( + new SearchHits(new SearchHit[] {}, new TotalHits(10000, TotalHits.Relation.EQUAL_TO), 1.0f) + ); + when(searchRequestBuilder.get()).thenReturn(searchResponse); + + ActionListener listener = mock(ActionListener.class); + rulePersistenceService.createRule(createRuleRequest, listener); + + ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(OpenSearchRejectedExecutionException.class); + verify(listener).onFailure(exceptionCaptor.capture()); + assertNotNull(exceptionCaptor.getValue()); + } + public void testConcurrentCreateDuplicateRules() throws InterruptedException { ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); int threadCount = 10; diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java index 9135b12c9cfaf..b8b6c5b73af90 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java @@ -75,6 +75,8 @@ import java.util.Map; import java.util.function.Supplier; +import static org.opensearch.rule.service.IndexStoredRulePersistenceService.MAX_WLM_RULES_SETTING; + /** * Plugin class for WorkloadManagement */ @@ -190,7 +192,11 @@ public List getRestHandlers( @Override public List> getSettings() { - return List.of(WorkloadGroupPersistenceService.MAX_QUERY_GROUP_COUNT, RefreshBasedSyncMechanism.RULE_SYNC_REFRESH_INTERVAL_SETTING); + return List.of( + WorkloadGroupPersistenceService.MAX_QUERY_GROUP_COUNT, + RefreshBasedSyncMechanism.RULE_SYNC_REFRESH_INTERVAL_SETTING, + MAX_WLM_RULES_SETTING + ); } @Override From 5d2e8a2e15766ae6dd6aea841a00bd53f4869f9f Mon Sep 17 00:00:00 2001 From: Ruirui Zhang Date: Wed, 9 Jul 2025 22:38:44 -0700 Subject: [PATCH 36/51] [WLM] Add WLM mode validation for workload group CRUD requests (#18652) Signed-off-by: Ruirui Zhang --- CHANGELOG.md | 1 + .../rest/WorkloadManagementRestIT.java | 32 ++++++++++ .../wlm/WlmClusterSettingValuesProvider.java | 64 +++++++++++++++++++ .../plugin/wlm/WorkloadManagementPlugin.java | 17 +++-- .../rest/RestCreateWorkloadGroupAction.java | 9 ++- .../rest/RestDeleteWorkloadGroupAction.java | 9 ++- .../rest/RestUpdateWorkloadGroupAction.java | 9 ++- .../rule/sync/RefreshBasedSyncMechanism.java | 30 ++------- .../WlmClusterSettingValuesProviderTests.java | 54 ++++++++++++++++ .../wlm/WorkloadManagementTestUtils.java | 14 ++++ .../RestCreateWorkloadGroupActionTests.java | 44 +++++++++++++ .../RestDeleteWorkloadGroupActionTests.java | 20 +++++- .../RestUpdateWorkloadGroupActionTests.java | 44 +++++++++++++ .../sync/RefreshBasedSyncMechanismTests.java | 30 ++++++--- .../test/wlm/10_workload_group.yml | 7 ++ 15 files changed, 339 insertions(+), 45 deletions(-) create mode 100644 plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WlmClusterSettingValuesProvider.java create mode 100644 plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WlmClusterSettingValuesProviderTests.java create mode 100644 plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestCreateWorkloadGroupActionTests.java create mode 100644 plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestUpdateWorkloadGroupActionTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index ac002af112f37..b76a33e010ccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add NodeResourceUsageStats to ClusterInfo ([#18480](https://github.com/opensearch-project/OpenSearch/issues/18472)) - Introduce SecureHttpTransportParameters experimental API (to complement SecureTransportParameters counterpart) ([#18572](https://github.com/opensearch-project/OpenSearch/issues/18572)) - Create equivalents of JSM's AccessController in the java agent ([#18346](https://github.com/opensearch-project/OpenSearch/issues/18346)) +- [WLM] Add WLM mode validation for workload group CRUD requests ([#18652](https://github.com/opensearch-project/OpenSearch/issues/18652)) - Introduced a new cluster-level API to fetch remote store metadata (segments and translogs) for each shard of an index. ([#18257](https://github.com/opensearch-project/OpenSearch/pull/18257)) - Add last index request timestamp columns to the `_cat/indices` API. ([10766](https://github.com/opensearch-project/OpenSearch/issues/10766)) - Introduce a new pull-based ingestion plugin for file-based indexing (for local testing) ([#18591](https://github.com/opensearch-project/OpenSearch/pull/18591)) diff --git a/plugins/workload-management/src/javaRestTest/java/org/opensearch/rest/WorkloadManagementRestIT.java b/plugins/workload-management/src/javaRestTest/java/org/opensearch/rest/WorkloadManagementRestIT.java index 00d3e901588b1..a1ade144a7701 100644 --- a/plugins/workload-management/src/javaRestTest/java/org/opensearch/rest/WorkloadManagementRestIT.java +++ b/plugins/workload-management/src/javaRestTest/java/org/opensearch/rest/WorkloadManagementRestIT.java @@ -13,11 +13,18 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.test.rest.OpenSearchRestTestCase; +import org.junit.Before; import java.io.IOException; +import java.util.Locale; public class WorkloadManagementRestIT extends OpenSearchRestTestCase { + @Before + public void enableWlmMode() throws Exception { + setWlmMode("enabled"); + } + public void testCreate() throws Exception { Response response = performOperation("PUT", "_wlm/workload_group", getCreateJson("analytics", "enforced", 0.4, 0.2)); assertEquals(response.getStatusLine().getStatusCode(), 200); @@ -129,6 +136,16 @@ public void testCRUD() throws Exception { performOperation("DELETE", "_wlm/workload_group/users3", null); } + public void testOperationWhenWlmDisabled() throws Exception { + setWlmMode("disabled"); + assertThrows( + ResponseException.class, + () -> performOperation("PUT", "_wlm/workload_group", getCreateJson("analytics", "enforced", 0.4, 0.2)) + ); + assertThrows(ResponseException.class, () -> performOperation("DELETE", "_wlm/workload_group/analytics4", null)); + assertOK(performOperation("GET", "_wlm/workload_group/", null)); + } + static String getCreateJson(String name, String resiliencyMode, double cpu, double memory) { return "{\n" + " \"name\": \"" @@ -171,4 +188,19 @@ Response performOperation(String method, String uriPath, String json) throws IOE } return client().performRequest(request); } + + private void setWlmMode(String mode) throws Exception { + String settingJson = String.format(Locale.ROOT, """ + { + "persistent": { + "wlm.workload_group.mode": "%s" + } + } + """, mode); + + Request request = new Request("PUT", "/_cluster/settings"); + request.setJsonEntity(settingJson); + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + } } diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WlmClusterSettingValuesProvider.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WlmClusterSettingValuesProvider.java new file mode 100644 index 0000000000000..56b78714c0db5 --- /dev/null +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WlmClusterSettingValuesProvider.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.wlm; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.wlm.WlmMode; +import org.opensearch.wlm.WorkloadManagementSettings; + +/** + * Central provider for maintaining and supplying the current values of wlm cluster settings. + * This class listens for updates to relevant settings and provides the latest setting values. + */ +public class WlmClusterSettingValuesProvider { + + private volatile WlmMode wlmMode; + + /** + * Constructor for WlmClusterSettingValuesProvider + * @param settings OpenSearch settings + * @param clusterSettings Cluster settings to register update listener + */ + public WlmClusterSettingValuesProvider(Settings settings, ClusterSettings clusterSettings) { + this.wlmMode = WorkloadManagementSettings.WLM_MODE_SETTING.get(settings); + clusterSettings.addSettingsUpdateConsumer(WorkloadManagementSettings.WLM_MODE_SETTING, this::setWlmMode); + } + + /** + * Check if WLM mode is ENABLED + * Throws an IllegalStateException if WLM mode is DISABLED or MONITOR ONLY. + * @param operationDescription A short text describing the operation, e.g. "create workload group". + */ + public void ensureWlmEnabled(String operationDescription) { + if (wlmMode != WlmMode.ENABLED) { + throw new IllegalStateException( + "Cannot " + + operationDescription + + " because workload management mode is disabled or monitor_only." + + "To enable this feature, set [wlm.workload_group.mode] to 'enabled' in cluster settings." + ); + } + } + + /** + * Set the latest WLM mode. + * @param mode The wlm mode to set + */ + private void setWlmMode(WlmMode mode) { + this.wlmMode = mode; + } + + /** + * Get the latest WLM mode. + */ + public WlmMode getWlmMode() { + return wlmMode; + } +} diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java index b8b6c5b73af90..fea81507633a1 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java @@ -93,6 +93,7 @@ public class WorkloadManagementPlugin extends Plugin implements ActionPlugin, Sy private static FeatureType featureType; private static RulePersistenceService rulePersistenceService; private static RuleRoutingService ruleRoutingService; + private WlmClusterSettingValuesProvider wlmClusterSettingValuesProvider; private AutoTaggingActionFilter autoTaggingActionFilter; /** @@ -114,6 +115,10 @@ public Collection createComponents( IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier ) { + wlmClusterSettingValuesProvider = new WlmClusterSettingValuesProvider( + clusterService.getSettings(), + clusterService.getClusterSettings() + ); featureType = new WorkloadGroupFeatureType(new WorkloadGroupFeatureValueValidator(clusterService)); RuleEntityParser parser = new XContentRuleParser(featureType); AttributeValueStoreFactory attributeValueStoreFactory = new AttributeValueStoreFactory( @@ -134,12 +139,10 @@ public Collection createComponents( RefreshBasedSyncMechanism refreshMechanism = new RefreshBasedSyncMechanism( threadPool, clusterService.getSettings(), - clusterService.getClusterSettings(), - parser, - ruleProcessingService, featureType, rulePersistenceService, - new RuleEventClassifier(Collections.emptySet(), ruleProcessingService) + new RuleEventClassifier(Collections.emptySet(), ruleProcessingService), + wlmClusterSettingValuesProvider ); autoTaggingActionFilter = new AutoTaggingActionFilter(ruleProcessingService, threadPool); @@ -183,10 +186,10 @@ public List getRestHandlers( Supplier nodesInCluster ) { return List.of( - new RestCreateWorkloadGroupAction(), + new RestCreateWorkloadGroupAction(wlmClusterSettingValuesProvider), new RestGetWorkloadGroupAction(), - new RestDeleteWorkloadGroupAction(), - new RestUpdateWorkloadGroupAction() + new RestDeleteWorkloadGroupAction(wlmClusterSettingValuesProvider), + new RestUpdateWorkloadGroupAction(wlmClusterSettingValuesProvider) ); } diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestCreateWorkloadGroupAction.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestCreateWorkloadGroupAction.java index 5ef59602f7893..5cb6a8a582df7 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestCreateWorkloadGroupAction.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestCreateWorkloadGroupAction.java @@ -11,6 +11,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.plugin.wlm.WlmClusterSettingValuesProvider; import org.opensearch.plugin.wlm.action.CreateWorkloadGroupAction; import org.opensearch.plugin.wlm.action.CreateWorkloadGroupRequest; import org.opensearch.plugin.wlm.action.CreateWorkloadGroupResponse; @@ -35,10 +36,15 @@ */ public class RestCreateWorkloadGroupAction extends BaseRestHandler { + private final WlmClusterSettingValuesProvider nonPluginSettingValuesProvider; + /** * Constructor for RestCreateWorkloadGroupAction + * @param nonPluginSettingValuesProvider the settings provider to access the current WLM mode */ - public RestCreateWorkloadGroupAction() {} + public RestCreateWorkloadGroupAction(WlmClusterSettingValuesProvider nonPluginSettingValuesProvider) { + this.nonPluginSettingValuesProvider = nonPluginSettingValuesProvider; + } @Override public String getName() { @@ -55,6 +61,7 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + nonPluginSettingValuesProvider.ensureWlmEnabled(getName()); try (XContentParser parser = request.contentParser()) { CreateWorkloadGroupRequest createWorkloadGroupRequest = CreateWorkloadGroupRequest.fromXContent(parser); return channel -> client.execute( diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestDeleteWorkloadGroupAction.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestDeleteWorkloadGroupAction.java index d0d82f43679fa..e1ad166ed6bda 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestDeleteWorkloadGroupAction.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestDeleteWorkloadGroupAction.java @@ -8,6 +8,7 @@ package org.opensearch.plugin.wlm.rest; +import org.opensearch.plugin.wlm.WlmClusterSettingValuesProvider; import org.opensearch.plugin.wlm.action.DeleteWorkloadGroupAction; import org.opensearch.plugin.wlm.action.DeleteWorkloadGroupRequest; import org.opensearch.rest.BaseRestHandler; @@ -27,10 +28,15 @@ */ public class RestDeleteWorkloadGroupAction extends BaseRestHandler { + private final WlmClusterSettingValuesProvider nonPluginSettingValuesProvider; + /** * Constructor for RestDeleteWorkloadGroupAction + * @param nonPluginSettingValuesProvider the settings provider to access the current WLM mode */ - public RestDeleteWorkloadGroupAction() {} + public RestDeleteWorkloadGroupAction(WlmClusterSettingValuesProvider nonPluginSettingValuesProvider) { + this.nonPluginSettingValuesProvider = nonPluginSettingValuesProvider; + } @Override public String getName() { @@ -47,6 +53,7 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + nonPluginSettingValuesProvider.ensureWlmEnabled(getName()); DeleteWorkloadGroupRequest deleteWorkloadGroupRequest = new DeleteWorkloadGroupRequest(request.param("name")); deleteWorkloadGroupRequest.clusterManagerNodeTimeout( request.paramAsTime("cluster_manager_timeout", deleteWorkloadGroupRequest.clusterManagerNodeTimeout()) diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestUpdateWorkloadGroupAction.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestUpdateWorkloadGroupAction.java index db77dc5963037..2e237cf191c75 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestUpdateWorkloadGroupAction.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestUpdateWorkloadGroupAction.java @@ -11,6 +11,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.plugin.wlm.WlmClusterSettingValuesProvider; import org.opensearch.plugin.wlm.action.UpdateWorkloadGroupAction; import org.opensearch.plugin.wlm.action.UpdateWorkloadGroupRequest; import org.opensearch.plugin.wlm.action.UpdateWorkloadGroupResponse; @@ -35,10 +36,15 @@ */ public class RestUpdateWorkloadGroupAction extends BaseRestHandler { + private final WlmClusterSettingValuesProvider nonPluginSettingValuesProvider; + /** * Constructor for RestUpdateWorkloadGroupAction + * @param nonPluginSettingValuesProvider the settings provider to access the current WLM mode */ - public RestUpdateWorkloadGroupAction() {} + public RestUpdateWorkloadGroupAction(WlmClusterSettingValuesProvider nonPluginSettingValuesProvider) { + this.nonPluginSettingValuesProvider = nonPluginSettingValuesProvider; + } @Override public String getName() { @@ -55,6 +61,7 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + nonPluginSettingValuesProvider.ensureWlmEnabled(getName()); try (XContentParser parser = request.contentParser()) { UpdateWorkloadGroupRequest updateWorkloadGroupRequest = UpdateWorkloadGroupRequest.fromXContent(parser, request.param("name")); return channel -> client.execute( diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/sync/RefreshBasedSyncMechanism.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/sync/RefreshBasedSyncMechanism.java index f367e63aa3f51..9bded4c845204 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/sync/RefreshBasedSyncMechanism.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/sync/RefreshBasedSyncMechanism.java @@ -11,15 +11,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; -import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; +import org.opensearch.plugin.wlm.WlmClusterSettingValuesProvider; import org.opensearch.plugin.wlm.rule.sync.detect.RuleEvent; import org.opensearch.plugin.wlm.rule.sync.detect.RuleEventClassifier; -import org.opensearch.rule.InMemoryRuleProcessingService; -import org.opensearch.rule.RuleEntityParser; import org.opensearch.rule.RulePersistenceService; import org.opensearch.rule.action.GetRuleRequest; import org.opensearch.rule.action.GetRuleResponse; @@ -28,7 +26,6 @@ import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; import org.opensearch.wlm.WlmMode; -import org.opensearch.wlm.WorkloadManagementSettings; import java.io.IOException; import java.util.Collections; @@ -65,12 +62,10 @@ public class RefreshBasedSyncMechanism extends AbstractLifecycleComponent { private final ThreadPool threadPool; private long refreshInterval; private volatile Scheduler.Cancellable scheduledFuture; - private final RuleEntityParser parser; - private final InMemoryRuleProcessingService ruleProcessingService; private final RulePersistenceService rulePersistenceService; private final RuleEventClassifier ruleEventClassifier; private final FeatureType featureType; - private WlmMode wlmMode; + private final WlmClusterSettingValuesProvider nonPluginSettingValuesProvider; // This var keeps the Rules which were present during last run of this service private Set lastRunIndexedRules; private static final Logger logger = LogManager.getLogger(RefreshBasedSyncMechanism.class); @@ -80,33 +75,26 @@ public class RefreshBasedSyncMechanism extends AbstractLifecycleComponent { * * @param threadPool * @param settings - * @param clusterSettings - * @param parser - * @param ruleProcessingService * @param featureType * @param rulePersistenceService * @param ruleEventClassifier + * @param nonPluginSettingValuesProvider */ public RefreshBasedSyncMechanism( ThreadPool threadPool, Settings settings, - ClusterSettings clusterSettings, - RuleEntityParser parser, - InMemoryRuleProcessingService ruleProcessingService, FeatureType featureType, RulePersistenceService rulePersistenceService, - RuleEventClassifier ruleEventClassifier + RuleEventClassifier ruleEventClassifier, + WlmClusterSettingValuesProvider nonPluginSettingValuesProvider ) { this.threadPool = threadPool; refreshInterval = RULE_SYNC_REFRESH_INTERVAL_SETTING.get(settings); - this.parser = parser; - this.ruleProcessingService = ruleProcessingService; this.featureType = featureType; this.rulePersistenceService = rulePersistenceService; this.lastRunIndexedRules = new HashSet<>(); this.ruleEventClassifier = ruleEventClassifier; - wlmMode = WorkloadManagementSettings.WLM_MODE_SETTING.get(settings); - clusterSettings.addSettingsUpdateConsumer(WorkloadManagementSettings.WLM_MODE_SETTING, this::setWlmMode); + this.nonPluginSettingValuesProvider = nonPluginSettingValuesProvider; } /** @@ -114,7 +102,7 @@ public RefreshBasedSyncMechanism( * but theoretically possible */ synchronized void doRun() { - if (wlmMode != WlmMode.ENABLED) { + if (nonPluginSettingValuesProvider.getWlmMode() != WlmMode.ENABLED) { return; } @@ -161,8 +149,4 @@ protected void doClose() throws IOException { scheduledFuture.cancel(); } } - - void setWlmMode(WlmMode mode) { - this.wlmMode = mode; - } } diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WlmClusterSettingValuesProviderTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WlmClusterSettingValuesProviderTests.java new file mode 100644 index 0000000000000..4dd4b423abc28 --- /dev/null +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WlmClusterSettingValuesProviderTests.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.wlm; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.wlm.WorkloadManagementSettings; +import org.junit.Before; + +import java.util.HashSet; + +import static org.mockito.Mockito.spy; + +public class WlmClusterSettingValuesProviderTests extends OpenSearchTestCase { + + private ClusterSettings clusterSettings; + + @Before + public void setUp() throws Exception { + super.setUp(); + try (WorkloadManagementPlugin plugin = new WorkloadManagementPlugin()) { + clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>(plugin.getSettings())); + clusterSettings.registerSetting(WorkloadManagementSettings.WLM_MODE_SETTING); + } + } + + public void testEnsureWlmEnabledThrowsWhenDisabled() { + WlmClusterSettingValuesProvider spyProvider = createSpyProviderWithMode("disabled"); + assertThrows(IllegalStateException.class, () -> spyProvider.ensureWlmEnabled("")); + } + + public void testEnsureWlmEnabledThrowsWhenMonitorOnly() { + WlmClusterSettingValuesProvider spyProvider = createSpyProviderWithMode("monitor_only"); + assertThrows(IllegalStateException.class, () -> spyProvider.ensureWlmEnabled("")); + } + + public void testEnsureWlmEnabledSucceedsWhenEnabled() { + WlmClusterSettingValuesProvider spyProvider = createSpyProviderWithMode("enabled"); + spyProvider.ensureWlmEnabled(""); + } + + private WlmClusterSettingValuesProvider createSpyProviderWithMode(String mode) { + Settings settings = Settings.builder().put(WorkloadManagementSettings.WLM_MODE_SETTING.getKey(), mode).build(); + WlmClusterSettingValuesProvider realProvider = new WlmClusterSettingValuesProvider(settings, clusterSettings); + return spy(realProvider); + } +} diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementTestUtils.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementTestUtils.java index bfc9ca356c112..60eee025e1c02 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementTestUtils.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementTestUtils.java @@ -19,10 +19,12 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.plugin.wlm.rule.sync.RefreshBasedSyncMechanism; import org.opensearch.plugin.wlm.service.WorkloadGroupPersistenceService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.wlm.MutableWorkloadGroupFragment; import org.opensearch.wlm.ResourceType; +import org.opensearch.wlm.WorkloadManagementSettings; import java.util.ArrayList; import java.util.Collection; @@ -165,4 +167,16 @@ public static void assertEqualWorkloadGroups( } } } + + public static WlmClusterSettingValuesProvider setUpNonPluginSettingValuesProvider(String wlmMode) throws Exception { + try (WorkloadManagementPlugin plugin = new WorkloadManagementPlugin()) { + Settings settings = Settings.builder() + .put(RefreshBasedSyncMechanism.RULE_SYNC_REFRESH_INTERVAL_SETTING_NAME, 1000) + .put(WorkloadManagementSettings.WLM_MODE_SETTING_NAME, wlmMode) + .build(); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>(plugin.getSettings())); + clusterSettings.registerSetting(WorkloadManagementSettings.WLM_MODE_SETTING); + return new WlmClusterSettingValuesProvider(settings, clusterSettings); + } + } } diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestCreateWorkloadGroupActionTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestCreateWorkloadGroupActionTests.java new file mode 100644 index 0000000000000..abd5135f5f529 --- /dev/null +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestCreateWorkloadGroupActionTests.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.wlm.rest; + +import org.opensearch.plugin.wlm.WlmClusterSettingValuesProvider; +import org.opensearch.plugin.wlm.WorkloadManagementTestUtils; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.transport.client.node.NodeClient; + +import static org.mockito.Mockito.mock; + +public class RestCreateWorkloadGroupActionTests extends OpenSearchTestCase { + + public void testPrepareRequestThrowsWhenWlmModeDisabled() { + try { + WlmClusterSettingValuesProvider nonPluginSettingValuesProvider = WorkloadManagementTestUtils + .setUpNonPluginSettingValuesProvider("disabled"); + RestCreateWorkloadGroupAction restCreateWorkloadGroupAction = new RestCreateWorkloadGroupAction(nonPluginSettingValuesProvider); + restCreateWorkloadGroupAction.prepareRequest(mock(RestRequest.class), mock(NodeClient.class)); + fail("Expected exception when WLM mode is DISABLED"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("create")); + } + } + + public void testPrepareRequestThrowsWhenWlmModeMonitorOnly() { + try { + WlmClusterSettingValuesProvider nonPluginSettingValuesProvider = WorkloadManagementTestUtils + .setUpNonPluginSettingValuesProvider("monitor_only"); + RestCreateWorkloadGroupAction restCreateWorkloadGroupAction = new RestCreateWorkloadGroupAction(nonPluginSettingValuesProvider); + restCreateWorkloadGroupAction.prepareRequest(mock(RestRequest.class), mock(NodeClient.class)); + fail("Expected exception when WLM mode is MONITOR_ONLY"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("create")); + } + } +} diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestDeleteWorkloadGroupActionTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestDeleteWorkloadGroupActionTests.java index 199b55754d132..c54313af62bae 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestDeleteWorkloadGroupActionTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestDeleteWorkloadGroupActionTests.java @@ -11,6 +11,8 @@ import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.unit.TimeValue; +import org.opensearch.plugin.wlm.WlmClusterSettingValuesProvider; +import org.opensearch.plugin.wlm.WorkloadManagementTestUtils; import org.opensearch.plugin.wlm.action.DeleteWorkloadGroupAction; import org.opensearch.plugin.wlm.action.DeleteWorkloadGroupRequest; import org.opensearch.rest.RestChannel; @@ -40,7 +42,7 @@ public class RestDeleteWorkloadGroupActionTests extends OpenSearchTestCase { * Test case to validate the construction for RestDeleteWorkloadGroupAction */ public void testConstruction() { - RestDeleteWorkloadGroupAction action = new RestDeleteWorkloadGroupAction(); + RestDeleteWorkloadGroupAction action = new RestDeleteWorkloadGroupAction(mock(WlmClusterSettingValuesProvider.class)); assertNotNull(action); assertEquals("delete_workload_group", action.getName()); List routes = action.routes(); @@ -55,7 +57,9 @@ public void testConstruction() { */ @SuppressWarnings("unchecked") public void testPrepareRequest() throws Exception { - RestDeleteWorkloadGroupAction restDeleteWorkloadGroupAction = new RestDeleteWorkloadGroupAction(); + RestDeleteWorkloadGroupAction restDeleteWorkloadGroupAction = new RestDeleteWorkloadGroupAction( + mock(WlmClusterSettingValuesProvider.class) + ); NodeClient nodeClient = mock(NodeClient.class); RestRequest realRequest = new FakeRestRequest(); realRequest.params().put("name", NAME_ONE); @@ -82,4 +86,16 @@ public void testPrepareRequest() throws Exception { any(RestToXContentListener.class) ); } + + public void testPrepareRequestThrowsWhenWlmModeDisabled() throws Exception { + try { + WlmClusterSettingValuesProvider nonPluginSettingValuesProvider = WorkloadManagementTestUtils + .setUpNonPluginSettingValuesProvider("disabled"); + RestDeleteWorkloadGroupAction restDeleteWorkloadGroupAction = new RestDeleteWorkloadGroupAction(nonPluginSettingValuesProvider); + restDeleteWorkloadGroupAction.prepareRequest(mock(RestRequest.class), mock(NodeClient.class)); + fail("Expected exception when WLM mode is DISABLED"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("delete")); + } + } } diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestUpdateWorkloadGroupActionTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestUpdateWorkloadGroupActionTests.java new file mode 100644 index 0000000000000..ba6ddb1a947a8 --- /dev/null +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestUpdateWorkloadGroupActionTests.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.wlm.rest; + +import org.opensearch.plugin.wlm.WlmClusterSettingValuesProvider; +import org.opensearch.plugin.wlm.WorkloadManagementTestUtils; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.transport.client.node.NodeClient; + +import static org.mockito.Mockito.mock; + +public class RestUpdateWorkloadGroupActionTests extends OpenSearchTestCase { + + public void testPrepareRequestThrowsWhenWlmModeDisabled() { + try { + WlmClusterSettingValuesProvider nonPluginSettingValuesProvider = WorkloadManagementTestUtils + .setUpNonPluginSettingValuesProvider("disabled"); + RestUpdateWorkloadGroupAction restUpdateWorkloadGroupAction = new RestUpdateWorkloadGroupAction(nonPluginSettingValuesProvider); + restUpdateWorkloadGroupAction.prepareRequest(mock(RestRequest.class), mock(NodeClient.class)); + fail("Expected exception when WLM mode is DISABLED"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("update")); + } + } + + public void testPrepareRequestThrowsWhenWlmModeMonitorOnly() { + try { + WlmClusterSettingValuesProvider nonPluginSettingValuesProvider = WorkloadManagementTestUtils + .setUpNonPluginSettingValuesProvider("monitor_only"); + RestUpdateWorkloadGroupAction restUpdateWorkloadGroupAction = new RestUpdateWorkloadGroupAction(nonPluginSettingValuesProvider); + restUpdateWorkloadGroupAction.prepareRequest(mock(RestRequest.class), mock(NodeClient.class)); + fail("Expected exception when WLM mode is MONITOR_ONLY"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("update")); + } + } +} diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rule/sync/RefreshBasedSyncMechanismTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rule/sync/RefreshBasedSyncMechanismTests.java index cca14c8778a87..739122e022bcb 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rule/sync/RefreshBasedSyncMechanismTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rule/sync/RefreshBasedSyncMechanismTests.java @@ -12,10 +12,10 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; import org.opensearch.plugin.wlm.AutoTaggingActionFilterTests; +import org.opensearch.plugin.wlm.WlmClusterSettingValuesProvider; import org.opensearch.plugin.wlm.WorkloadManagementPlugin; import org.opensearch.plugin.wlm.rule.sync.detect.RuleEventClassifier; import org.opensearch.rule.InMemoryRuleProcessingService; -import org.opensearch.rule.RuleEntityParser; import org.opensearch.rule.RulePersistenceService; import org.opensearch.rule.action.GetRuleRequest; import org.opensearch.rule.action.GetRuleResponse; @@ -24,12 +24,10 @@ import org.opensearch.rule.autotagging.Rule; import org.opensearch.rule.storage.AttributeValueStoreFactory; import org.opensearch.rule.storage.DefaultAttributeValueStore; -import org.opensearch.rule.storage.XContentRuleParser; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.client.Client; -import org.opensearch.wlm.WlmMode; import org.opensearch.wlm.WorkloadManagementSettings; import java.io.IOException; @@ -59,6 +57,8 @@ public class RefreshBasedSyncMechanismTests extends OpenSearchTestCase { Scheduler.Cancellable scheduledFuture; RuleEventClassifier ruleEventClassifier; FeatureType featureType; + WlmClusterSettingValuesProvider nonPluginSettingValuesProvider; + ClusterSettings clusterSettings; @Override public void setUp() throws Exception { @@ -68,7 +68,7 @@ public void setUp() throws Exception { .put(RefreshBasedSyncMechanism.RULE_SYNC_REFRESH_INTERVAL_SETTING_NAME, 1000) .put(WorkloadManagementSettings.WLM_MODE_SETTING_NAME, "enabled") .build(); - ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>(plugin.getSettings())); + clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>(plugin.getSettings())); clusterSettings.registerSetting(WorkloadManagementSettings.WLM_MODE_SETTING); featureType = mock(FeatureType.class); mockThreadPool = mock(ThreadPool.class); @@ -76,7 +76,7 @@ public void setUp() throws Exception { rulePersistenceService = mock(RulePersistenceService.class); ruleEventClassifier = new RuleEventClassifier(Collections.emptySet(), ruleProcessingService); attributeValueStoreFactory = new AttributeValueStoreFactory(featureType, DefaultAttributeValueStore::new); - RuleEntityParser parser = new XContentRuleParser(featureType); + nonPluginSettingValuesProvider = new WlmClusterSettingValuesProvider(settings, clusterSettings); mockClient = mock(Client.class); scheduledFuture = mock(Scheduler.Cancellable.class); when(mockThreadPool.scheduleWithFixedDelay(any(), any(), any())).thenReturn(scheduledFuture); @@ -84,12 +84,10 @@ public void setUp() throws Exception { sut = new RefreshBasedSyncMechanism( mockThreadPool, settings, - clusterSettings, - parser, - ruleProcessingService, featureType, rulePersistenceService, - ruleEventClassifier + ruleEventClassifier, + nonPluginSettingValuesProvider ); } } @@ -123,7 +121,19 @@ public void testDoClose() throws IOException { */ @SuppressWarnings("unchecked") public void testDoRunWhenWLM_isDisabled() { - sut.setWlmMode(WlmMode.DISABLED); + Settings disabledSettings = Settings.builder() + .put(RefreshBasedSyncMechanism.RULE_SYNC_REFRESH_INTERVAL_SETTING_NAME, 1000) + .put(WorkloadManagementSettings.WLM_MODE_SETTING_NAME, "disabled") + .build(); + WlmClusterSettingValuesProvider disabledWlmModeProvider = new WlmClusterSettingValuesProvider(disabledSettings, clusterSettings); + sut = new RefreshBasedSyncMechanism( + mockThreadPool, + disabledSettings, + featureType, + rulePersistenceService, + ruleEventClassifier, + disabledWlmModeProvider + ); sut.doRun(); verify(rulePersistenceService, times(0)).getRule(any(GetRuleRequest.class), any(ActionListener.class)); } diff --git a/plugins/workload-management/src/yamlRestTest/resources/rest-api-spec/test/wlm/10_workload_group.yml b/plugins/workload-management/src/yamlRestTest/resources/rest-api-spec/test/wlm/10_workload_group.yml index 178639638890d..a9ba5d300c9fa 100644 --- a/plugins/workload-management/src/yamlRestTest/resources/rest-api-spec/test/wlm/10_workload_group.yml +++ b/plugins/workload-management/src/yamlRestTest/resources/rest-api-spec/test/wlm/10_workload_group.yml @@ -3,6 +3,13 @@ version: " - 2.16.99" reason: "QueryGroup WorkloadManagement feature was added in 2.17" + - do: + cluster.put_settings: + flat_settings: true + body: + transient: + wlm.workload_group.mode: "enabled" + - do: create_workload_group_context: body: From 57d1d17c8b72494d0f1c0b029c8355bddf5116fd Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Thu, 10 Jul 2025 12:53:33 +0530 Subject: [PATCH 37/51] Allow upload of translog-1 generation while migrating to remote (#18713) Signed-off-by: Gaurav Bafna --- .../index/translog/transfer/BlobStoreTransferService.java | 6 +++--- .../org/opensearch/indices/recovery/RecoveryTarget.java | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java index a0ac50709bf7e..81438b978cd99 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java @@ -40,7 +40,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import static org.opensearch.common.blobstore.BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC; @@ -192,7 +191,8 @@ private void uploadBlob( listener.onFailure(new FileTransferException(fileSnapshot, ex)); }); - Objects.requireNonNull(fileSnapshot.getChecksum()); + // Only the first generation doesn't have checksum + assert (fileSnapshot.getChecksum() != null || fileSnapshot.getName().contains("-1.")); uploadBlobAsyncInternal( fileSnapshot.getName(), fileSnapshot.getName(), @@ -226,7 +226,7 @@ void uploadBlobAsyncInternal( BlobPath blobPath, WritePriority writePriority, RemoteTransferContainer.OffsetRangeInputStreamSupplier inputStreamSupplier, - long expectedChecksum, + Long expectedChecksum, ActionListener completionListener, Map metadata ) throws IOException { diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java index f47b082de3856..1c386c1c1623e 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java @@ -224,8 +224,7 @@ public void prepareForTranslogOperations(int totalTranslogOps, ActionListener { indexShard.refresh("remote store migration"); }); indexShard.waitForRemoteStoreSync(this::setLastAccessTime); logger.info("Remote Store is now seeded for {}", indexShard.shardId()); From d6c857f311528e6f82f89b2de40cf3d52b3bb555 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Mon, 14 Jul 2025 10:59:01 +0800 Subject: [PATCH 38/51] Fix max_score is null when using _score as a primary sort (#18715) * Fix max_score is null when sorting on score firstly Signed-off-by: Binlong Gao * modify changelog Signed-off-by: Binlong Gao * Fix test failure Signed-off-by: Binlong Gao --------- Signed-off-by: Binlong Gao --- CHANGELOG.md | 1 + .../test/search/400_max_score.yml | 89 +++++++++++++++++++ .../search/query/TopDocsCollectorContext.java | 20 ++++- .../search/query/QueryPhaseTests.java | 43 +++++++++ 4 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/400_max_score.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index b76a33e010ccf..34374c82333c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix the visit of sub queries for HasParentQuery and HasChildQuery ([#18621](https://github.com/opensearch-project/OpenSearch/pull/18621)) - Fix the backward compatibility regression with COMPLEMENT for Regexp queries introduced in OpenSearch 3.0 ([#18640](https://github.com/opensearch-project/OpenSearch/pull/18640)) - Fix Replication lag computation ([#18602](https://github.com/opensearch-project/OpenSearch/pull/18602)) +- Fix max_score is null when sorting on score firstly ([#18715](https://github.com/opensearch-project/OpenSearch/pull/18715)) - Fixed Staggered merge - load average replace with AverageTrackers, some Default thresholds modified ([#18666](https://github.com/opensearch-project/OpenSearch/pull/18666)) ### Security diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/400_max_score.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/400_max_score.yml new file mode 100644 index 0000000000000..f81aafb3150de --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/400_max_score.yml @@ -0,0 +1,89 @@ +setup: + - do: + indices.create: + index: test_1 + body: + mappings: + properties: + foo: + type: keyword + + - do: + bulk: + refresh: true + body: + - index: + _index: test_1 + - foo: bar + - index: + _index: test_1 + - foo: bar + + - do: + indices.refresh: + index: [test_1] + +--- +teardown: + - do: + indices.delete: + index: test_1 + +# related issue: https://github.com/opensearch-project/OpenSearch/issues/18714 +--- +"Test max score with sorting on score firstly": + - skip: + version: " - 3.2.0" + reason: Fixed in 3.2.0 + + - do: + search: + index: test_1 + body: + query: { term: { foo: bar} } + sort: [{ _score: desc }, { _doc: desc }] + - match: { hits.total: 2 } + - length: { hits.hits: 2 } + - match: { max_score: 1.0 } + + - do: + search: + index: test_1 + body: + query: { term: { foo: bar} } + sort: [{ _score: asc }, { _doc: desc }] + - match: { hits.total: 2 } + - length: { hits.hits: 2 } + - match: { max_score: null } + +--- +"Test max score with sorting on score firstly with concurrent segment search enabled": + - skip: + version: " - 3.2.0" + reason: Fixed in 3.2.0 + + - do: + indices.put_settings: + index: test_1 + body: + index.search.concurrent_segment_search.mode: 'all' + + - do: + search: + index: test_1 + body: + query: { term: { foo: bar} } + sort: [{ _score: desc }, { _doc: desc }] + - match: { hits.total: 2 } + - length: { hits.hits: 2 } + - match: { max_score: 1.0 } + + - do: + search: + index: test_1 + body: + query: { term: { foo: bar} } + sort: [{ _score: asc }, { _doc: desc }] + - match: { hits.total: 2 } + - length: { hits.hits: 2 } + - match: { max_score: null } diff --git a/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java b/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java index d60780bebf7e7..674f0e50c436a 100644 --- a/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java +++ b/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java @@ -450,6 +450,16 @@ private SimpleTopDocsCollectorContext( return topDocs.scoreDocs[0].score; } }; + } else if (SortField.FIELD_SCORE.equals(sortAndFormats.sort.getSort()[0])) { + maxScoreSupplier = () -> { + TopDocs topDocs = topDocsSupplier.get(); + if (topDocs.scoreDocs.length == 0) { + return Float.NaN; + } else { + FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[0]; + return (float) fieldDoc.fields[0]; + } + }; } else if (trackMaxScore) { maxScoreCollector = new MaxScoreCollector(); maxScoreSupplier = maxScoreCollector::getMaxScore; @@ -595,8 +605,14 @@ TopDocsAndMaxScore newTopDocs(final TopDocs topDocs, final float maxScore, final newTopDocs = new TopDocs(totalHits, scoreDocs); } - if (Float.isNaN(maxScore) && newTopDocs.scoreDocs.length > 0 && sortAndFormats == null) { - return new TopDocsAndMaxScore(newTopDocs, newTopDocs.scoreDocs[0].score); + if (Float.isNaN(maxScore) && newTopDocs.scoreDocs.length > 0) { + float maxScoreFromDoc = maxScore; + if (sortAndFormats == null) { + maxScoreFromDoc = newTopDocs.scoreDocs[0].score; + } else if (SortField.FIELD_SCORE.equals(sortAndFormats.sort.getSort()[0])) { + maxScoreFromDoc = (float) ((FieldDoc) newTopDocs.scoreDocs[0]).fields[0]; + } + return new TopDocsAndMaxScore(newTopDocs, maxScoreFromDoc); } else { return new TopDocsAndMaxScore(newTopDocs, maxScore); } diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java index 73333ea2c5706..ae32ebd0a6f7a 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java @@ -994,6 +994,49 @@ public void testMinScore() throws Exception { dir.close(); } + public void testMaxScoreWithSortOnScoreFirstly() throws Exception { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + + final int numDocs = scaledRandomIntBetween(10, 20); + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(new StringField("foo", "bar", Store.NO)); + doc.add(new StringField("filter", "f1" + ((i > 0) ? " " + i : ""), Store.NO)); + w.addDocument(doc); + } + w.close(); + + IndexReader reader = DirectoryReader.open(dir); + TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader, executor)); + context.trackScores(false); + Sort sort = new Sort(new SortField(null, SortField.Type.SCORE), new SortField(null, SortField.Type.DOC)); + SortAndFormats sortAndFormats = new SortAndFormats(sort, new DocValueFormat[] { DocValueFormat.RAW, DocValueFormat.RAW }); + context.sort(sortAndFormats); + context.parsedQuery( + new ParsedQuery( + new BooleanQuery.Builder().add(new TermQuery(new Term("foo", "bar")), Occur.MUST) + .add(new TermQuery(new Term("filter", "f1")), Occur.SHOULD) + .build() + ) + ); + context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap())); + context.setSize(1); + context.trackTotalHitsUpTo(5); + + QueryPhase.executeInternal(context.withCleanQueryResult(), queryPhaseSearcher); + assertFalse(Float.isNaN(context.queryResult().getMaxScore())); + assertEquals(1, context.queryResult().topDocs().topDocs.scoreDocs.length); + assertThat( + ((FieldDoc) context.queryResult().topDocs().topDocs.scoreDocs[0]).fields[0], + equalTo(context.queryResult().getMaxScore()) + ); + + reader.close(); + dir.close(); + } + public void testMaxScore() throws Exception { Directory dir = newDirectory(); final Sort sort = new Sort(new SortField("filter", SortField.Type.STRING)); From acf8aa1d1f615304356f5879de042a88d8d8c7d3 Mon Sep 17 00:00:00 2001 From: Aditi Goyal <31741158+Adiiigo@users.noreply.github.com> Date: Mon, 14 Jul 2025 11:04:36 +0530 Subject: [PATCH 39/51] Changes for the Upload Workflow for merged segments (#18610) * Changes for the Upload Workflow for merged segments Signed-off-by: Aditi Goyal * Addressed Todos Signed-off-by: Aditi Goyal * Added Test Cases Signed-off-by: Aditi Goyal * Removed the usage of Forbidden APIs Signed-off-by: Aditi Goyal * Updated the code to include priority upload as paramter Signed-off-by: Aditi Goyal * Updated the remote listener to use remote upploader interface Signed-off-by: Aditi Goyal * Added JavaDocs across the changes Signed-off-by: Aditi Goyal * Addressed spotless check Signed-off-by: Aditi Goyal * Dummy commit(can be reverted) Signed-off-by: Aditi Goyal * Revert "Dummy commit(can be reverted)" This reverts commit e542b99704e12112f0522f390585164b86ecb18e. Signed-off-by: Aditi Goyal --------- Signed-off-by: Aditi Goyal Co-authored-by: Aditi Goyal --- .../shard/RemoteStoreRefreshListener.java | 72 ++- .../index/shard/RemoteStoreUploader.java | 30 ++ .../shard/RemoteStoreUploaderService.java | 89 ++++ .../RemoteStoreUploaderServiceTests.java | 477 ++++++++++++++++++ 4 files changed, 625 insertions(+), 43 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/shard/RemoteStoreUploader.java create mode 100644 server/src/main/java/org/opensearch/index/shard/RemoteStoreUploaderService.java create mode 100644 server/src/test/java/org/opensearch/index/shard/RemoteStoreUploaderServiceTests.java diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 8ace4848806d7..f3ee23a7505d2 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -9,9 +9,7 @@ package org.opensearch.index.shard; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; @@ -19,7 +17,6 @@ import org.apache.lucene.store.IndexInput; import org.opensearch.action.LatchedActionListener; import org.opensearch.action.bulk.BackoffPolicy; -import org.opensearch.action.support.GroupedActionListener; import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.logging.Loggers; @@ -30,7 +27,6 @@ import org.opensearch.index.engine.InternalEngine; import org.opensearch.index.remote.RemoteSegmentTransferTracker; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.store.CompositeDirectory; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.translog.Translog; @@ -49,6 +45,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; import java.util.stream.Collectors; import static org.opensearch.index.seqno.SequenceNumbers.LOCAL_CHECKPOINT_KEY; @@ -93,6 +90,7 @@ public final class RemoteStoreRefreshListener extends ReleasableRetryableRefresh private volatile Iterator backoffDelayIterator; private final SegmentReplicationCheckpointPublisher checkpointPublisher; private final RemoteStoreSettings remoteStoreSettings; + private final RemoteStoreUploader remoteStoreUploader; public RemoteStoreRefreshListener( IndexShard indexShard, @@ -106,6 +104,7 @@ public RemoteStoreRefreshListener( this.storeDirectory = indexShard.store().directory(); this.remoteDirectory = (RemoteSegmentStoreDirectory) ((FilterDirectory) ((FilterDirectory) indexShard.remoteStore().directory()) .getDelegate()).getDelegate(); + remoteStoreUploader = new RemoteStoreUploaderService(indexShard, storeDirectory, remoteDirectory); localSegmentChecksumMap = new HashMap<>(); RemoteSegmentMetadata remoteSegmentMetadata = null; if (indexShard.routingEntry().primary()) { @@ -324,6 +323,32 @@ public void onFailure(Exception e) { return successful.get(); } + /** + * Uploads new segment files to the remote store. + * + * @param localSegmentsPostRefresh collection of segment files present after refresh + * @param localSegmentsSizeMap map of segment file names to their sizes + * @param segmentUploadsCompletedListener listener to be notified when upload completes + */ + private void uploadNewSegments( + Collection localSegmentsPostRefresh, + Map localSegmentsSizeMap, + ActionListener segmentUploadsCompletedListener + ) { + Collection filteredFiles = localSegmentsPostRefresh.stream().filter(file -> !skipUpload(file)).collect(Collectors.toList()); + Function, UploadListener> uploadListenerFunction = (Map sizeMap) -> createUploadListener( + localSegmentsSizeMap + ); + + remoteStoreUploader.uploadSegments( + filteredFiles, + localSegmentsSizeMap, + segmentUploadsCompletedListener, + uploadListenerFunction, + isLowPriorityUpload() + ); + } + /** * Clears the stale files from the latest local segment checksum map. * @@ -424,45 +449,6 @@ void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos se } } - private void uploadNewSegments( - Collection localSegmentsPostRefresh, - Map localSegmentsSizeMap, - ActionListener listener - ) { - Collection filteredFiles = localSegmentsPostRefresh.stream().filter(file -> !skipUpload(file)).collect(Collectors.toList()); - if (filteredFiles.size() == 0) { - logger.debug("No new segments to upload in uploadNewSegments"); - listener.onResponse(null); - return; - } - - logger.debug("Effective new segments files to upload {}", filteredFiles); - ActionListener> mappedListener = ActionListener.map(listener, resp -> null); - GroupedActionListener batchUploadListener = new GroupedActionListener<>(mappedListener, filteredFiles.size()); - Directory directory = ((FilterDirectory) (((FilterDirectory) storeDirectory).getDelegate())).getDelegate(); - - for (String src : filteredFiles) { - // Initializing listener here to ensure that the stats increment operations are thread-safe - UploadListener statsListener = createUploadListener(localSegmentsSizeMap); - ActionListener aggregatedListener = ActionListener.wrap(resp -> { - statsListener.onSuccess(src); - batchUploadListener.onResponse(resp); - if (directory instanceof CompositeDirectory) { - ((CompositeDirectory) directory).afterSyncToRemote(src); - } - }, ex -> { - logger.warn(() -> new ParameterizedMessage("Exception: [{}] while uploading segment files", ex), ex); - if (ex instanceof CorruptIndexException) { - indexShard.failShard(ex.getMessage(), ex); - } - statsListener.onFailure(src); - batchUploadListener.onFailure(ex); - }); - statsListener.beforeUpload(src); - remoteDirectory.copyFrom(storeDirectory, src, IOContext.DEFAULT, aggregatedListener, isLowPriorityUpload()); - } - } - boolean isLowPriorityUpload() { return isLocalOrSnapshotRecoveryOrSeeding(); } diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreUploader.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreUploader.java new file mode 100644 index 0000000000000..1a60aabf3e609 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreUploader.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.shard; + +import org.opensearch.common.util.UploadListener; +import org.opensearch.core.action.ActionListener; + +import java.util.Collection; +import java.util.Map; +import java.util.function.Function; + +/** + * Interface to handle the functionality for uploading data in the remote store + */ +public interface RemoteStoreUploader { + + void uploadSegments( + Collection localSegments, + Map localSegmentsSizeMap, + ActionListener listener, + Function, UploadListener> uploadListenerFunction, + boolean isLowPriorityUpload + ); +} diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreUploaderService.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreUploaderService.java new file mode 100644 index 0000000000000..7cab0b258f107 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreUploaderService.java @@ -0,0 +1,89 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.shard; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FilterDirectory; +import org.apache.lucene.store.IOContext; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.common.logging.Loggers; +import org.opensearch.common.util.UploadListener; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.store.CompositeDirectory; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; + +import java.util.Collection; +import java.util.Map; +import java.util.function.Function; + +/** + * The service essentially acts as a bridge between local segment storage and remote storage, + * ensuring efficient and reliable segment synchronization while providing comprehensive monitoring and error handling. + */ +public class RemoteStoreUploaderService implements RemoteStoreUploader { + + private final Logger logger; + + private final IndexShard indexShard; + private final Directory storeDirectory; + private final RemoteSegmentStoreDirectory remoteDirectory; + + public RemoteStoreUploaderService(IndexShard indexShard, Directory storeDirectory, RemoteSegmentStoreDirectory remoteDirectory) { + logger = Loggers.getLogger(getClass(), indexShard.shardId()); + this.indexShard = indexShard; + this.storeDirectory = storeDirectory; + this.remoteDirectory = remoteDirectory; + } + + @Override + public void uploadSegments( + Collection localSegments, + Map localSegmentsSizeMap, + ActionListener listener, + Function, UploadListener> uploadListenerFunction, + boolean isLowPriorityUpload + ) { + if (localSegments.isEmpty()) { + logger.debug("No new segments to upload in uploadNewSegments"); + listener.onResponse(null); + return; + } + + logger.debug("Effective new segments files to upload {}", localSegments); + ActionListener> mappedListener = ActionListener.map(listener, resp -> null); + GroupedActionListener batchUploadListener = new GroupedActionListener<>(mappedListener, localSegments.size()); + Directory directory = ((FilterDirectory) (((FilterDirectory) storeDirectory).getDelegate())).getDelegate(); + + for (String localSegment : localSegments) { + // Initializing listener here to ensure that the stats increment operations are thread-safe + UploadListener statsListener = uploadListenerFunction.apply(localSegmentsSizeMap); + ActionListener aggregatedListener = ActionListener.wrap(resp -> { + statsListener.onSuccess(localSegment); + batchUploadListener.onResponse(resp); + // Once uploaded to Remote, local files become eligible for eviction from FileCache + if (directory instanceof CompositeDirectory) { + ((CompositeDirectory) directory).afterSyncToRemote(localSegment); + } + }, ex -> { + logger.warn(() -> new ParameterizedMessage("Exception: [{}] while uploading segment files", ex), ex); + if (ex instanceof CorruptIndexException) { + indexShard.failShard(ex.getMessage(), ex); + } + statsListener.onFailure(localSegment); + batchUploadListener.onFailure(ex); + }); + statsListener.beforeUpload(localSegment); + // Place where the actual upload is happening + remoteDirectory.copyFrom(storeDirectory, localSegment, IOContext.DEFAULT, aggregatedListener, isLowPriorityUpload); + } + } +} diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreUploaderServiceTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreUploaderServiceTests.java new file mode 100644 index 0000000000000..d522020d0198b --- /dev/null +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreUploaderServiceTests.java @@ -0,0 +1,477 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.shard; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FilterDirectory; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.common.util.UploadListener; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.store.CompositeDirectory; +import org.opensearch.index.store.RemoteDirectory; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Test class for {@link RemoteStoreUploaderService}. + * Tests various scenarios of segment upload functionality including success cases, + * error handling, and different directory configurations. + */ +public class RemoteStoreUploaderServiceTests extends OpenSearchTestCase { + + /** Mock IndexShard instance used across tests */ + private IndexShard mockIndexShard; + + /** Mock Directory representing the local store directory */ + private Directory mockStoreDirectory; + + /** Mock RemoteSegmentStoreDirectory for remote storage operations */ + private RemoteSegmentStoreDirectory mockRemoteDirectory; + + /** The service under test */ + private RemoteStoreUploaderService uploaderService; + + /** Mock upload listener for tracking upload events */ + private UploadListener mockUploadListener; + + /** Mock function that creates upload listeners */ + private Function, UploadListener> mockUploadListenerFunction; + + /** + * Sets up the test environment before each test method. + * Initializes all mock objects and the service under test. + * + * @throws Exception if setup fails + */ + @Override + public void setUp() throws Exception { + super.setUp(); + mockIndexShard = mock(IndexShard.class); + ShardId shardId = new ShardId(new Index("test", "test"), 0); + when(mockIndexShard.shardId()).thenReturn(shardId); + + // Mock IndexShard methods instead of setting private fields + when(mockIndexShard.state()).thenReturn(IndexShardState.STARTED); + mockStoreDirectory = mock(FilterDirectory.class); + // Use a real instance with mocked dependencies instead of mocking the final class + RemoteDirectory remoteDataDirectory = mock(RemoteDirectory.class); + mockRemoteDirectory = createMockRemoteDirectory(remoteDataDirectory); + mockUploadListener = mock(UploadListener.class); + mockUploadListenerFunction = mock(Function.class); + + when(mockUploadListenerFunction.apply(any())).thenReturn(mockUploadListener); + + uploaderService = new RemoteStoreUploaderService(mockIndexShard, mockStoreDirectory, mockRemoteDirectory); + } + + /** + * Creates a real RemoteSegmentStoreDirectory instance with mocked dependencies + * instead of trying to mock the final class directly. + * This approach is used because RemoteSegmentStoreDirectory is a final class + * that cannot be mocked directly. + * + * @param remoteDirectory the remote directory to use (currently unused) + * @return a new RemoteSegmentStoreDirectory instance with mocked dependencies + * @throws RuntimeException if the directory creation fails + */ + private RemoteSegmentStoreDirectory createMockRemoteDirectory(RemoteDirectory remoteDirectory) { + try { + RemoteDirectory remoteDataDirectory = mock(RemoteDirectory.class); + RemoteDirectory remoteMetadataDirectory = mock(RemoteDirectory.class); + RemoteStoreLockManager lockManager = mock(RemoteStoreLockManager.class); + ThreadPool threadPool = mock(ThreadPool.class); + ShardId shardId = new ShardId(new Index("test", "test"), 0); + + return new RemoteSegmentStoreDirectory(remoteDataDirectory, remoteMetadataDirectory, lockManager, threadPool, shardId); + } catch (IOException e) { + throw new RuntimeException("Failed to create mock RemoteSegmentStoreDirectory", e); + } + } + + /** + * Tests that uploading an empty collection of segments completes successfully + * without performing any actual upload operations. + * + * @throws Exception if the test fails + */ + public void testUploadSegmentsWithEmptyCollection() throws Exception { + Collection emptySegments = Collections.emptyList(); + Map segmentSizeMap = new HashMap<>(); + CountDownLatch latch = new CountDownLatch(1); + + ActionListener listener = ActionListener.wrap( + response -> latch.countDown(), + exception -> fail("Should not fail for empty segments") + ); + + uploaderService.uploadSegments(emptySegments, segmentSizeMap, listener, mockUploadListenerFunction, false); + + assertTrue(latch.await(1, TimeUnit.SECONDS)); + } + + /** + * Tests successful segment upload with low priority upload flag set to false. + * Verifies that segments are uploaded correctly and upload listeners are notified. + * + * @throws Exception if the test fails + */ + public void testUploadSegmentsSuccessWithHighPriorityUpload() throws Exception { + Collection segments = Arrays.asList("segment1", "segment2"); + Map segmentSizeMap = new HashMap<>(); + segmentSizeMap.put("segment1", 100L); + segmentSizeMap.put("segment2", 200L); + + // Create a fresh mock IndexShard + IndexShard freshMockShard = mock(IndexShard.class); + ShardId shardId = new ShardId(new Index("test", "test"), 1); + when(freshMockShard.shardId()).thenReturn(shardId); + when(freshMockShard.state()).thenReturn(IndexShardState.STARTED); + + // Create a mock directory structure that matches what the code expects + Directory innerMockDelegate = mock(Directory.class); + FilterDirectory innerFilterDirectory = new TestFilterDirectory(new TestFilterDirectory(innerMockDelegate)); + + FilterDirectory outerFilterDirectory = new TestFilterDirectory(new TestFilterDirectory(innerFilterDirectory)); + + // Setup the real RemoteSegmentStoreDirectory to handle copyFrom calls + RemoteDirectory remoteDirectory = mock(RemoteDirectory.class); + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory( + remoteDirectory, + mock(RemoteDirectory.class), + mock(RemoteStoreLockManager.class), + freshMockShard.getThreadPool(), + freshMockShard.shardId() + ); + + // Create a new uploader service with the fresh mocks + RemoteStoreUploaderService testUploaderService = new RemoteStoreUploaderService( + freshMockShard, + outerFilterDirectory, + remoteSegmentStoreDirectory + ); + + doAnswer(invocation -> { + ActionListener callback = invocation.getArgument(5); + callback.onResponse(null); + return true; + }).when(remoteDirectory).copyFrom(any(), any(), any(), any(), any(), any(), any(Boolean.class)); + + CountDownLatch latch = new CountDownLatch(1); + + ActionListener listener = ActionListener.wrap( + response -> latch.countDown(), + exception -> fail("Upload should succeed: " + exception.getMessage()) + ); + + testUploaderService.uploadSegments(segments, segmentSizeMap, listener, mockUploadListenerFunction, false); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + // Verify the upload listener was called correctly + verify(mockUploadListener, times(2)).beforeUpload(any(String.class)); + verify(mockUploadListener, times(2)).onSuccess(any(String.class)); + } + + /** + * Tests successful segment upload with low priority upload flag set to true. + * Verifies that segments are uploaded correctly and upload listeners are notified. + * + * @throws Exception if the test fails + */ + public void testUploadSegmentsSuccessWithLowPriorityUpload() throws Exception { + Collection segments = Arrays.asList("segment1", "segment2"); + Map segmentSizeMap = new HashMap<>(); + segmentSizeMap.put("segment1", 100L); + segmentSizeMap.put("segment2", 200L); + + // Create a fresh mock IndexShard + IndexShard freshMockShard = mock(IndexShard.class); + ShardId shardId = new ShardId(new Index("test", "test"), 1); + when(freshMockShard.shardId()).thenReturn(shardId); + when(freshMockShard.state()).thenReturn(IndexShardState.STARTED); + + // Create a mock directory structure that matches what the code expects + Directory innerMockDelegate = mock(Directory.class); + FilterDirectory innerFilterDirectory = new TestFilterDirectory(new TestFilterDirectory(innerMockDelegate)); + + FilterDirectory outerFilterDirectory = new TestFilterDirectory(new TestFilterDirectory(innerFilterDirectory)); + + // Setup the real RemoteSegmentStoreDirectory to handle copyFrom calls + RemoteDirectory remoteDirectory = mock(RemoteDirectory.class); + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory( + remoteDirectory, + mock(RemoteDirectory.class), + mock(RemoteStoreLockManager.class), + freshMockShard.getThreadPool(), + freshMockShard.shardId() + ); + + // Create a new uploader service with the fresh mocks + RemoteStoreUploaderService testUploaderService = new RemoteStoreUploaderService( + freshMockShard, + outerFilterDirectory, + remoteSegmentStoreDirectory + ); + + doAnswer(invocation -> { + ActionListener callback = invocation.getArgument(5); + callback.onResponse(null); + return true; + }).when(remoteDirectory).copyFrom(any(), any(), any(), any(), any(), any(), any(Boolean.class)); + + CountDownLatch latch = new CountDownLatch(1); + + ActionListener listener = ActionListener.wrap( + response -> latch.countDown(), + exception -> fail("Upload should succeed: " + exception.getMessage()) + ); + + testUploaderService.uploadSegments(segments, segmentSizeMap, listener, mockUploadListenerFunction, true); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + // Verify the upload listener was called correctly + verify(mockUploadListener, times(2)).beforeUpload(any(String.class)); + verify(mockUploadListener, times(2)).onSuccess(any(String.class)); + } + + /** + * Tests segment upload functionality when using a CompositeDirectory. + * Verifies that the afterSyncToRemote callback is invoked on the CompositeDirectory + * after successful upload operations. + * + * @throws Exception if the test fails + */ + public void testUploadSegmentsWithCompositeDirectory() throws Exception { + Collection segments = Arrays.asList("segment1"); + Map segmentSizeMap = new HashMap<>(); + segmentSizeMap.put("segment1", 100L); + + // Create a fresh mock IndexShard + IndexShard freshMockShard = mock(IndexShard.class); + ShardId shardId = new ShardId(new Index("test", "test"), 0); + when(freshMockShard.shardId()).thenReturn(shardId); + + // Create a mock ShardRouting and set it as a field on the IndexShard mock + ShardRouting mockShardRouting = mock(ShardRouting.class); + freshMockShard.shardRouting = mockShardRouting; + when(mockShardRouting.primary()).thenReturn(true); + + CompositeDirectory mockCompositeDirectory = mock(CompositeDirectory.class); + FilterDirectory innerFilterDirectory = new TestFilterDirectory(mockCompositeDirectory); + FilterDirectory outerFilterDirectory = new TestFilterDirectory(innerFilterDirectory); + + // Setup the real RemoteSegmentStoreDirectory to handle copyFrom calls + RemoteDirectory remoteDirectory = mock(RemoteDirectory.class); + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory( + remoteDirectory, + mock(RemoteDirectory.class), + mock(RemoteStoreLockManager.class), + freshMockShard.getThreadPool(), + freshMockShard.shardId() + ); + + // Create a new uploader service with the fresh mocks + RemoteStoreUploaderService testUploaderService = new RemoteStoreUploaderService( + freshMockShard, + outerFilterDirectory, + remoteSegmentStoreDirectory + ); + + // Setup the real RemoteSegmentStoreDirectory to handle copyFrom calls + doAnswer(invocation -> { + ActionListener callback = invocation.getArgument(5); + callback.onResponse(null); + return true; + }).when(remoteDirectory).copyFrom(any(), any(), any(), any(), any(), any(), any(Boolean.class)); + + CountDownLatch latch = new CountDownLatch(1); + + ActionListener listener = ActionListener.wrap( + response -> latch.countDown(), + exception -> fail("Upload should succeed: " + exception.getMessage()) + ); + + testUploaderService.uploadSegments(segments, segmentSizeMap, listener, mockUploadListenerFunction, false); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + verify(mockCompositeDirectory).afterSyncToRemote("segment1"); + } + + /** + * Tests error handling when a CorruptIndexException occurs during segment upload. + * Verifies that the shard is failed with the appropriate error message + * and the upload listener is notified of the failure. + * + * @throws Exception if the test fails + */ + public void testUploadSegmentsWithCorruptIndexException() throws Exception { + Collection segments = Arrays.asList("segment1"); + Map segmentSizeMap = new HashMap<>(); + segmentSizeMap.put("segment1", 100L); + + // Create a fresh mock IndexShard + IndexShard freshMockShard = mock(IndexShard.class); + ShardId shardId = new ShardId(new Index("test", "test"), 0); + when(freshMockShard.shardId()).thenReturn(shardId); + + // Create a mock ShardRouting and set it as a field on the IndexShard mock + ShardRouting mockShardRouting = mock(ShardRouting.class); + freshMockShard.shardRouting = mockShardRouting; + when(mockShardRouting.primary()).thenReturn(true); + + Directory innerMockDelegate = mock(Directory.class); + FilterDirectory innerFilterDirectory = new TestFilterDirectory(new TestFilterDirectory(innerMockDelegate)); + + FilterDirectory outerFilterDirectory = new TestFilterDirectory(new TestFilterDirectory(innerFilterDirectory)); + + // Setup the real RemoteSegmentStoreDirectory to handle copyFrom calls + RemoteDirectory remoteDirectory = mock(RemoteDirectory.class); + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory( + remoteDirectory, + mock(RemoteDirectory.class), + mock(RemoteStoreLockManager.class), + freshMockShard.getThreadPool(), + freshMockShard.shardId() + ); + + // Create a new uploader service with the fresh mocks + RemoteStoreUploaderService testUploaderService = new RemoteStoreUploaderService( + freshMockShard, + outerFilterDirectory, + remoteSegmentStoreDirectory + ); + + CorruptIndexException corruptException = new CorruptIndexException("Index corrupted", "test"); + CountDownLatch latch = new CountDownLatch(1); + + // Setup the real RemoteSegmentStoreDirectory to handle copyFrom calls + RemoteDirectory remoteDataDirectory = mock(RemoteDirectory.class); + doAnswer(invocation -> { + ActionListener callback = invocation.getArgument(5); + callback.onFailure(corruptException); + return true; + }).when(remoteDirectory).copyFrom(any(), any(), any(), any(), any(), any(), any(Boolean.class)); + + ActionListener listener = ActionListener.wrap(response -> fail("Should not succeed with corrupt index"), exception -> { + assertEquals(corruptException, exception); + latch.countDown(); + }); + + testUploaderService.uploadSegments(segments, segmentSizeMap, listener, mockUploadListenerFunction, false); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + verify(freshMockShard).failShard(eq("Index corrupted (resource=test)"), eq(corruptException)); + verify(mockUploadListener).onFailure("segment1"); + } + + /** + * Tests error handling when a generic RuntimeException occurs during segment upload. + * Verifies that the shard is NOT failed (unlike CorruptIndexException) + * but the upload listener is still notified of the failure. + * + * @throws Exception if the test fails + */ + public void testUploadSegmentsWithGenericException() throws Exception { + Collection segments = Arrays.asList("segment1"); + Map segmentSizeMap = new HashMap<>(); + segmentSizeMap.put("segment1", 100L); + + // Create a fresh mock IndexShard + IndexShard freshMockShard = mock(IndexShard.class); + ShardId shardId = new ShardId(new Index("test", "test"), 0); + when(freshMockShard.shardId()).thenReturn(shardId); + + // Create a mock ShardRouting and set it as a field on the IndexShard mock + ShardRouting mockShardRouting = mock(ShardRouting.class); + freshMockShard.shardRouting = mockShardRouting; + when(mockShardRouting.primary()).thenReturn(true); + + Directory innerMockDelegate = mock(Directory.class); + FilterDirectory innerFilterDirectory = new TestFilterDirectory(new TestFilterDirectory(innerMockDelegate)); + + FilterDirectory outerFilterDirectory = new TestFilterDirectory(new TestFilterDirectory(innerFilterDirectory)); + + // Setup the real RemoteSegmentStoreDirectory to handle copyFrom calls + RemoteDirectory remoteDirectory = mock(RemoteDirectory.class); + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory( + remoteDirectory, + mock(RemoteDirectory.class), + mock(RemoteStoreLockManager.class), + freshMockShard.getThreadPool(), + freshMockShard.shardId() + ); + + // Create a new uploader service with the fresh mocks + RemoteStoreUploaderService testUploaderService = new RemoteStoreUploaderService( + freshMockShard, + outerFilterDirectory, + remoteSegmentStoreDirectory + ); + + RuntimeException genericException = new RuntimeException("Generic error"); + CountDownLatch latch = new CountDownLatch(1); + + // Setup the real RemoteSegmentStoreDirectory to handle copyFrom calls + doAnswer(invocation -> { + ActionListener callback = invocation.getArgument(5); + callback.onFailure(genericException); + return true; + }).when(remoteDirectory).copyFrom(any(), any(), any(), any(), any(), any(), any(Boolean.class)); + + ActionListener listener = ActionListener.wrap(response -> fail("Should not succeed with generic exception"), exception -> { + assertEquals(genericException, exception); + latch.countDown(); + }); + + testUploaderService.uploadSegments(segments, segmentSizeMap, listener, mockUploadListenerFunction, false); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + verify(freshMockShard, never()).failShard(any(), any()); + verify(mockUploadListener).onFailure("segment1"); + } + + /** + * Test implementation of FilterDirectory used for creating nested directory structures + * in tests. This class simply delegates all operations to the wrapped directory. + */ + public static class TestFilterDirectory extends FilterDirectory { + + /** + * Creates a new TestFilterDirectory wrapping the given directory. + * + * @param in the directory to wrap + */ + public TestFilterDirectory(Directory in) { + super(in); + } + } +} From 5944bfa72ad85e26bfcd6b14e45e412268d2f513 Mon Sep 17 00:00:00 2001 From: guojialiang Date: Mon, 14 Jul 2025 15:27:04 +0800 Subject: [PATCH 40/51] Modify merged segment pre-copy classes licenses (#18737) Signed-off-by: guojialiang --- .../engine/LocalMergedSegmentWarmer.java | 24 ------------------- .../engine/MergedSegmentWarmerFactory.java | 24 ------------------- .../RemoteStoreMergedSegmentWarmer.java | 24 ------------------- .../MergedSegmentReplicationTarget.java | 24 ------------------- .../checkpoint/MergeSegmentCheckpoint.java | 24 ------------------- 5 files changed, 120 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/engine/LocalMergedSegmentWarmer.java b/server/src/main/java/org/opensearch/index/engine/LocalMergedSegmentWarmer.java index 33f1a2b547b78..a9c23fd1f6c5b 100644 --- a/server/src/main/java/org/opensearch/index/engine/LocalMergedSegmentWarmer.java +++ b/server/src/main/java/org/opensearch/index/engine/LocalMergedSegmentWarmer.java @@ -6,30 +6,6 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.index.engine; import org.apache.lucene.index.IndexWriter; diff --git a/server/src/main/java/org/opensearch/index/engine/MergedSegmentWarmerFactory.java b/server/src/main/java/org/opensearch/index/engine/MergedSegmentWarmerFactory.java index eda93d22d9c3b..3e118ff6cc020 100644 --- a/server/src/main/java/org/opensearch/index/engine/MergedSegmentWarmerFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/MergedSegmentWarmerFactory.java @@ -6,30 +6,6 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.index.engine; import org.apache.lucene.index.IndexWriter; diff --git a/server/src/main/java/org/opensearch/index/engine/RemoteStoreMergedSegmentWarmer.java b/server/src/main/java/org/opensearch/index/engine/RemoteStoreMergedSegmentWarmer.java index f3d5f8a4cc93b..9a34e43423dbd 100644 --- a/server/src/main/java/org/opensearch/index/engine/RemoteStoreMergedSegmentWarmer.java +++ b/server/src/main/java/org/opensearch/index/engine/RemoteStoreMergedSegmentWarmer.java @@ -6,30 +6,6 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.index.engine; import org.apache.lucene.index.IndexWriter; diff --git a/server/src/main/java/org/opensearch/indices/replication/MergedSegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/MergedSegmentReplicationTarget.java index 75cd3985ffc88..00ced0cfe4798 100644 --- a/server/src/main/java/org/opensearch/indices/replication/MergedSegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/MergedSegmentReplicationTarget.java @@ -6,30 +6,6 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.indices.replication; import org.opensearch.action.StepListener; diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/MergeSegmentCheckpoint.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/MergeSegmentCheckpoint.java index 424a2cc98939e..15b90c4fb8dfe 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/MergeSegmentCheckpoint.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/MergeSegmentCheckpoint.java @@ -6,30 +6,6 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.indices.replication.checkpoint; import org.opensearch.common.annotation.ExperimentalApi; From 43cd6b59331055d6c7fcfc5bea5d7d6145b7a3f1 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Mon, 14 Jul 2025 08:42:25 -0700 Subject: [PATCH 41/51] `Lucene.java` Code Improvement (#18710) * Code Improvement Signed-off-by: Prudhvi Godithi * Code Improvement with a new method Signed-off-by: Prudhvi Godithi --------- Signed-off-by: Prudhvi Godithi --- .../org/opensearch/common/lucene/Lucene.java | 77 +++++++------------ 1 file changed, 26 insertions(+), 51 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/lucene/Lucene.java b/server/src/main/java/org/opensearch/common/lucene/Lucene.java index 4845ee9ae09c9..47427b08b6207 100644 --- a/server/src/main/java/org/opensearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/opensearch/common/lucene/Lucene.java @@ -357,63 +357,38 @@ public static TopDocsAndMaxScore readTopDocs(StreamInput in) throws IOException public static FieldDoc readFieldDoc(StreamInput in) throws IOException { Comparable[] cFields = new Comparable[in.readVInt()]; for (int j = 0; j < cFields.length; j++) { - byte type = in.readByte(); - if (type == 0) { - cFields[j] = null; - } else if (type == 1) { - cFields[j] = in.readString(); - } else if (type == 2) { - cFields[j] = in.readInt(); - } else if (type == 3) { - cFields[j] = in.readLong(); - } else if (type == 4) { - cFields[j] = in.readFloat(); - } else if (type == 5) { - cFields[j] = in.readDouble(); - } else if (type == 6) { - cFields[j] = in.readByte(); - } else if (type == 7) { - cFields[j] = in.readShort(); - } else if (type == 8) { - cFields[j] = in.readBoolean(); - } else if (type == 9) { - cFields[j] = in.readBytesRef(); - } else if (type == 10) { - cFields[j] = new BigInteger(in.readString()); - } else { - throw new IOException("Can't match type [" + type + "]"); - } + cFields[j] = readTypedValue(in); } return new FieldDoc(in.readVInt(), in.readFloat(), cFields); } public static Comparable readSortValue(StreamInput in) throws IOException { + return readTypedValue(in); + } + + /** + * Reads a typed value from the stream based on a type byte prefix. + * + * @param in the input stream + * @return the deserialized Comparable value + * @throws IOException if reading fails or type is unknown + */ + private static Comparable readTypedValue(StreamInput in) throws IOException { byte type = in.readByte(); - if (type == 0) { - return null; - } else if (type == 1) { - return in.readString(); - } else if (type == 2) { - return in.readInt(); - } else if (type == 3) { - return in.readLong(); - } else if (type == 4) { - return in.readFloat(); - } else if (type == 5) { - return in.readDouble(); - } else if (type == 6) { - return in.readByte(); - } else if (type == 7) { - return in.readShort(); - } else if (type == 8) { - return in.readBoolean(); - } else if (type == 9) { - return in.readBytesRef(); - } else if (type == 10) { - return new BigInteger(in.readString()); - } else { - throw new IOException("Can't match type [" + type + "]"); - } + return switch (type) { + case 0 -> null; + case 1 -> in.readString(); + case 2 -> in.readInt(); + case 3 -> in.readLong(); + case 4 -> in.readFloat(); + case 5 -> in.readDouble(); + case 6 -> in.readByte(); + case 7 -> in.readShort(); + case 8 -> in.readBoolean(); + case 9 -> in.readBytesRef(); + case 10 -> new BigInteger(in.readString()); + default -> throw new IOException("Can't match type [" + type + "]"); + }; } public static ScoreDoc readScoreDoc(StreamInput in) throws IOException { From 3cda34186e4d5b86637d91989c5de529fdd747bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jul 2025 16:08:22 -0400 Subject: [PATCH 42/51] Bump org.apache.commons:commons-lang3 from 3.17.0 to 3.18.0 in /plugins/repository-hdfs (#18745) * Bump org.apache.commons:commons-lang3 in /plugins/repository-hdfs Bumps org.apache.commons:commons-lang3 from 3.17.0 to 3.18.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-lang3 dependency-version: 3.18.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 1 + plugins/repository-hdfs/build.gradle | 2 +- plugins/repository-hdfs/licenses/commons-lang3-3.17.0.jar.sha1 | 1 - plugins/repository-hdfs/licenses/commons-lang3-3.18.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-hdfs/licenses/commons-lang3-3.17.0.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/commons-lang3-3.18.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 34374c82333c8..b2c33fbc41329 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.bouncycastle:bcpg-fips` from 2.0.10 to 2.0.11 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) - Bump `com.password4j:password4j` from 1.8.2 to 1.8.3 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) - Bump `com.azure:azure-core` from 1.55.3 to 1.55.5 ([#18691](https://github.com/opensearch-project/OpenSearch/pull/18691)) +- Bump `org.apache.commons:commons-lang3` from 3.17.0 to 3.18.0 ([#18745](https://github.com/opensearch-project/OpenSearch/pull/18745)) ### Deprecated diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index bd0257243bec1..c73fd2e57ee34 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -72,7 +72,7 @@ dependencies { api "org.apache.commons:commons-compress:${versions.commonscompress}" api 'org.apache.commons:commons-configuration2:2.12.0' api "commons-io:commons-io:${versions.commonsio}" - api 'org.apache.commons:commons-lang3:3.17.0' + api 'org.apache.commons:commons-lang3:3.18.0' implementation 'com.google.re2j:re2j:1.8' api 'javax.servlet:servlet-api:2.5' api "org.slf4j:slf4j-api:${versions.slf4j}" diff --git a/plugins/repository-hdfs/licenses/commons-lang3-3.17.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-lang3-3.17.0.jar.sha1 deleted file mode 100644 index 073922fda1dbe..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-lang3-3.17.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b17d2136f0460dcc0d2016ceefca8723bdf4ee70 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-lang3-3.18.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-lang3-3.18.0.jar.sha1 new file mode 100644 index 0000000000000..a1a6598bd4f1b --- /dev/null +++ b/plugins/repository-hdfs/licenses/commons-lang3-3.18.0.jar.sha1 @@ -0,0 +1 @@ +fb14946f0e39748a6571de0635acbe44e7885491 \ No newline at end of file From b2b88d279e0d09fb2c7b6efdde0a4cc4c698b950 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 10:28:07 +0800 Subject: [PATCH 43/51] Bump org.apache.logging.log4j:log4j-core from 2.25.0 to 2.25.1 in /buildSrc/src/testKit/thirdPartyAudit/sample_jars (#18744) * Bump org.apache.logging.log4j:log4j-core Bumps org.apache.logging.log4j:log4j-core from 2.25.0 to 2.25.1. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-version: 2.25.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 +- buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2c33fbc41329..1fa792d72d5a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,7 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `stefanzweifel/git-auto-commit-action` from 5 to 6 ([#18524](https://github.com/opensearch-project/OpenSearch/pull/18524)) - Bump Apache Lucene to 10.2.2 ([#18573](https://github.com/opensearch-project/OpenSearch/pull/18573)) -- Bump `org.apache.logging.log4j:log4j-core` from 2.24.3 to 2.25.0 ([#18589](https://github.com/opensearch-project/OpenSearch/pull/18589)) +- Bump `org.apache.logging.log4j:log4j-core` from 2.24.3 to 2.25.1 ([#18589](https://github.com/opensearch-project/OpenSearch/pull/18589), [#18744](https://github.com/opensearch-project/OpenSearch/pull/18744)) - Bump `com.google.code.gson:gson` from 2.13.0 to 2.13.1 ([#18585](https://github.com/opensearch-project/OpenSearch/pull/18585)) - Bump `com.azure:azure-core-http-netty` from 1.15.11 to 1.15.12 ([#18586](https://github.com/opensearch-project/OpenSearch/pull/18586)) - Bump `com.squareup.okio:okio` from 3.13.0 to 3.15.0 ([#18645](https://github.com/opensearch-project/OpenSearch/pull/18645), [#18689](https://github.com/opensearch-project/OpenSearch/pull/18689)) diff --git a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle index b88501b0b80cb..3a44aa603378c 100644 --- a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle +++ b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle @@ -17,7 +17,7 @@ repositories { } dependencies { - implementation "org.apache.logging.log4j:log4j-core:2.25.0" + implementation "org.apache.logging.log4j:log4j-core:2.25.1" } ["0.0.1", "0.0.2"].forEach { v -> From e09d6428f03921e68e514a4f4184b6ededd3b087 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 12:03:33 +0800 Subject: [PATCH 44/51] Bump com.azure:azure-storage-common from 12.29.0 to 12.29.1 in /plugins/repository-azure (#18742) * Bump com.azure:azure-storage-common in /plugins/repository-azure Bumps [com.azure:azure-storage-common](https://github.com/Azure/azure-sdk-for-java) from 12.29.0 to 12.29.1. - [Release notes](https://github.com/Azure/azure-sdk-for-java/releases) - [Commits](https://github.com/Azure/azure-sdk-for-java/compare/azure-storage-blob_12.29.0...azure-storage-blob_12.29.1) --- updated-dependencies: - dependency-name: com.azure:azure-storage-common dependency-version: 12.29.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- .../licenses/azure-storage-common-12.29.0.jar.sha1 | 1 - .../licenses/azure-storage-common-12.29.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/azure-storage-common-12.29.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-storage-common-12.29.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fa792d72d5a0..3ef940b7a5429 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.bouncycastle:bcpg-fips` from 2.0.10 to 2.0.11 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) - Bump `com.password4j:password4j` from 1.8.2 to 1.8.3 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) - Bump `com.azure:azure-core` from 1.55.3 to 1.55.5 ([#18691](https://github.com/opensearch-project/OpenSearch/pull/18691)) +- Bump `com.azure:azure-storage-common` from 12.29.0 to 12.29.1 ([#18742](https://github.com/opensearch-project/OpenSearch/pull/18742)) - Bump `org.apache.commons:commons-lang3` from 3.17.0 to 3.18.0 ([#18745](https://github.com/opensearch-project/OpenSearch/pull/18745)) ### Deprecated diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 3576f28a57c32..9975e15ae3ff1 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -47,7 +47,7 @@ dependencies { api 'com.azure:azure-core:1.55.5' api 'com.azure:azure-json:1.5.0' api 'com.azure:azure-xml:1.2.0' - api 'com.azure:azure-storage-common:12.29.0' + api 'com.azure:azure-storage-common:12.29.1' api 'com.azure:azure-core-http-netty:1.15.12' api "io.netty:netty-codec-dns:${versions.netty}" api "io.netty:netty-codec-socks:${versions.netty}" diff --git a/plugins/repository-azure/licenses/azure-storage-common-12.29.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-common-12.29.0.jar.sha1 deleted file mode 100644 index 6de0e8a945011..0000000000000 --- a/plugins/repository-azure/licenses/azure-storage-common-12.29.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -900fefe982179300c239fbe661e6135a760f5ee6 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-common-12.29.1.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-common-12.29.1.jar.sha1 new file mode 100644 index 0000000000000..5bfb37b06f137 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-storage-common-12.29.1.jar.sha1 @@ -0,0 +1 @@ +d4151d507125bfb255287bfde5d4ab27cd35e478 \ No newline at end of file From 624f85a9fcf63006517907c758482ef164bd228d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 14:58:00 +0800 Subject: [PATCH 45/51] Bump com.google.jimfs:jimfs from 1.3.0 to 1.3.1 in /distribution/tools/plugin-cli (#18743) * Bump com.google.jimfs:jimfs in /distribution/tools/plugin-cli Bumps [com.google.jimfs:jimfs](https://github.com/google/jimfs) from 1.3.0 to 1.3.1. - [Release notes](https://github.com/google/jimfs/releases) - [Commits](https://github.com/google/jimfs/compare/v1.3.0...v1.3.1) --- updated-dependencies: - dependency-name: com.google.jimfs:jimfs dependency-version: 1.3.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + distribution/tools/plugin-cli/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ef940b7a5429..d04c01edc47d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.bouncycastle:bcpg-fips` from 2.0.10 to 2.0.11 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) - Bump `com.password4j:password4j` from 1.8.2 to 1.8.3 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668)) - Bump `com.azure:azure-core` from 1.55.3 to 1.55.5 ([#18691](https://github.com/opensearch-project/OpenSearch/pull/18691)) +- Bump `com.google.jimfs:jimfs` from 1.3.0 to 1.3.1 ([#18743](https://github.com/opensearch-project/OpenSearch/pull/18743)) - Bump `com.azure:azure-storage-common` from 12.29.0 to 12.29.1 ([#18742](https://github.com/opensearch-project/OpenSearch/pull/18742)) - Bump `org.apache.commons:commons-lang3` from 3.17.0 to 3.18.0 ([#18745](https://github.com/opensearch-project/OpenSearch/pull/18745)) diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index a239926ae56f3..8beb17bb8bf9a 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -41,7 +41,7 @@ dependencies { api "org.bouncycastle:bc-fips:${versions.bouncycastle_jce}" api "org.bouncycastle:bcpg-fips:${versions.bouncycastle_pg}" testImplementation project(":test:framework") - testImplementation 'com.google.jimfs:jimfs:1.3.0' + testImplementation 'com.google.jimfs:jimfs:1.3.1' testRuntimeOnly("com.google.guava:guava:${versions.guava}") { transitive = false } From 8f69dcf4b0227bc599f95bf5303105468e0d2eca Mon Sep 17 00:00:00 2001 From: rayshrey <121871912+rayshrey@users.noreply.github.com> Date: Tue, 15 Jul 2025 12:30:21 +0530 Subject: [PATCH 46/51] Fix sync method to account for block files in Composite Directory (#18660) Signed-off-by: Shreyansh Ray --- .../opensearch/index/store/CompositeDirectory.java | 14 ++++++++++---- .../index/store/CompositeDirectoryTests.java | 6 ++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java b/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java index 99eb1db04b296..99e70dd215b5b 100644 --- a/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java @@ -249,10 +249,16 @@ public IndexOutput createOutput(String name, IOContext context) throws IOExcepti public void sync(Collection names) throws IOException { ensureOpen(); logger.trace("Composite Directory[{}]: sync() called {}", this::toString, () -> names); - Collection remoteFiles = Arrays.asList(getRemoteFiles()); - Collection filesToSync = names.stream().filter(name -> remoteFiles.contains(name) == false).collect(Collectors.toList()); - logger.trace("Composite Directory[{}]: Synced files : {}", this::toString, () -> filesToSync); - localDirectory.sync(filesToSync); + Set remoteFiles = Set.of(getRemoteFiles()); + Set localFilesHavingBlocks = Arrays.stream(listLocalFiles()) + .filter(FileTypeUtils::isBlockFile) + .map(file -> file.substring(0, file.indexOf(BLOCK_FILE_IDENTIFIER))) + .collect(Collectors.toSet()); + Collection fullFilesToSync = names.stream() + .filter(name -> (remoteFiles.contains(name) == false) && (localFilesHavingBlocks.contains(name) == false)) + .collect(Collectors.toList()); + logger.trace("Composite Directory[{}]: Synced files : {}", this::toString, () -> fullFilesToSync); + localDirectory.sync(fullFilesToSync); } /** diff --git a/server/src/test/java/org/opensearch/index/store/CompositeDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/CompositeDirectoryTests.java index 531bca97df662..7ee00712a236d 100644 --- a/server/src/test/java/org/opensearch/index/store/CompositeDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/CompositeDirectoryTests.java @@ -132,6 +132,12 @@ public void testSync() throws IOException { // All the files in the below list are present either locally or on remote, so sync should work as expected Collection names = List.of("_0.cfe", "_0.cfs", "_0.si", "_1.cfe", "_2.cfe", "segments_1"); compositeDirectory.sync(names); + // Deleting file _1.cfe and then adding its blocks locally so that full file is not present but block files are present in local + // State of _1.cfe file after these operations - not present in remote, full file not present locally but blocks present in local + compositeDirectory.deleteFile("_1.cfe"); + addFilesToDirectory(new String[] { "_1.cfe_block_0", "_1.cfe_block_2" }); + // Sync should work as expected since blocks are present in local + compositeDirectory.sync(List.of("_1.cfe")); // Below list contains a non-existent file, hence will throw an error Collection names1 = List.of("_0.cfe", "_0.cfs", "_0.si", "_1.cfe", "_2.cfe", "segments_1", "non_existent_file"); assertThrows(NoSuchFileException.class, () -> compositeDirectory.sync(names1)); From 4ad96863177a9b6b78845b017ea8551928fa2bc7 Mon Sep 17 00:00:00 2001 From: Adrian-Kim <110332047+BeomSeogKim@users.noreply.github.com> Date: Wed, 16 Jul 2025 02:22:32 +0900 Subject: [PATCH 47/51] [Cleanup] Remove upgrade-cli tool and related build references (#18494) The `upgrade-cli` tool is deprecated and no longer needed. - Removed: distribution/tools/upgrade-cli - Cleaned up: settings.gradle, distribution/build.gradle - Deleted: distribution/src/bin/opensearch-upgrade(.bat) Signed-off-by: BeomSeogKim --- .github/dependabot.yml | 8 - distribution/build.gradle | 6 +- distribution/src/bin/opensearch-upgrade | 7 - distribution/src/bin/opensearch-upgrade.bat | 16 -- distribution/tools/upgrade-cli/build.gradle | 35 --- .../upgrade-cli/licenses/jackson-LICENSE | 8 - .../tools/upgrade-cli/licenses/jackson-NOTICE | 20 -- .../jackson-annotations-2.18.2.jar.sha1 | 1 - .../licenses/jackson-databind-2.18.2.jar.sha1 | 1 - .../cli/upgrade/DetectEsInstallationTask.java | 203 ----------------- .../cli/upgrade/ImportJvmOptionsTask.java | 50 ---- .../tools/cli/upgrade/ImportKeystoreTask.java | 67 ------ .../upgrade/ImportLog4jPropertiesTask.java | 81 ------- .../cli/upgrade/ImportYmlConfigTask.java | 108 --------- .../tools/cli/upgrade/InstallPluginsTask.java | 118 ---------- .../cli/upgrade/KeystoreWrapperUtil.java | 33 --- .../tools/cli/upgrade/TaskInput.java | 108 --------- .../tools/cli/upgrade/UpgradeCli.java | 67 ------ .../tools/cli/upgrade/UpgradeTask.java | 41 ---- .../tools/cli/upgrade/ValidateInputTask.java | 87 ------- .../tools/cli/upgrade/package-info.java | 12 - .../DetectEsInstallationTaskTests.java | 67 ------ .../ImportLog4jPropertiesTaskTests.java | 80 ------- .../cli/upgrade/ImportYmlConfigTaskTests.java | 78 ------- .../cli/upgrade/InstallPluginsTaskTests.java | 78 ------- .../tools/cli/upgrade/UpgradeCliTests.java | 160 ------------- .../cli/upgrade/ValidateInputTaskTests.java | 69 ------ .../resources/config/elasticsearch.keystore | Bin 286 -> 0 bytes .../test/resources/config/elasticsearch.yml | 8 - .../config/jvm.options.d/test.options | 2 - .../test/resources/config/log4j2.properties | 213 ------------------ settings.gradle | 1 - 32 files changed, 1 insertion(+), 1832 deletions(-) delete mode 100755 distribution/src/bin/opensearch-upgrade delete mode 100644 distribution/src/bin/opensearch-upgrade.bat delete mode 100644 distribution/tools/upgrade-cli/build.gradle delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-LICENSE delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-NOTICE delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-annotations-2.18.2.jar.sha1 delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-databind-2.18.2.jar.sha1 delete mode 100644 distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/DetectEsInstallationTask.java delete mode 100644 distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportJvmOptionsTask.java delete mode 100644 distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportKeystoreTask.java delete mode 100644 distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportLog4jPropertiesTask.java delete mode 100644 distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportYmlConfigTask.java delete mode 100644 distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/InstallPluginsTask.java delete mode 100644 distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/KeystoreWrapperUtil.java delete mode 100644 distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/TaskInput.java delete mode 100644 distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/UpgradeCli.java delete mode 100644 distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/UpgradeTask.java delete mode 100644 distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ValidateInputTask.java delete mode 100644 distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/package-info.java delete mode 100644 distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/DetectEsInstallationTaskTests.java delete mode 100644 distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/ImportLog4jPropertiesTaskTests.java delete mode 100644 distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/ImportYmlConfigTaskTests.java delete mode 100644 distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/InstallPluginsTaskTests.java delete mode 100644 distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/UpgradeCliTests.java delete mode 100644 distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/ValidateInputTaskTests.java delete mode 100644 distribution/tools/upgrade-cli/src/test/resources/config/elasticsearch.keystore delete mode 100644 distribution/tools/upgrade-cli/src/test/resources/config/elasticsearch.yml delete mode 100644 distribution/tools/upgrade-cli/src/test/resources/config/jvm.options.d/test.options delete mode 100644 distribution/tools/upgrade-cli/src/test/resources/config/log4j2.properties diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a5dd25d2910f2..28c0d861158b8 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -445,14 +445,6 @@ updates: labels: - "dependabot" - "dependencies" - - directory: /distribution/tools/upgrade-cli/ - open-pull-requests-limit: 1 - package-ecosystem: gradle - schedule: - interval: weekly - labels: - - "dependabot" - - "dependencies" - directory: /doc-tools/ open-pull-requests-limit: 1 package-ecosystem: gradle diff --git a/distribution/build.gradle b/distribution/build.gradle index d6ad3fb96931c..5de6fa0611ea0 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -312,7 +312,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { * Properties to expand when copying packaging files * *****************************************************************************/ configurations { - ['libs', 'libsPluginCli', 'libsKeystoreCli', 'libsUpgradeCli', 'bcFips'].each { + ['libs', 'libsPluginCli', 'libsKeystoreCli', 'bcFips'].each { create(it) { canBeConsumed = false canBeResolved = true @@ -333,7 +333,6 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { libsPluginCli project(':distribution:tools:plugin-cli') libsKeystoreCli project(path: ':distribution:tools:keystore-cli') - libsUpgradeCli project(path: ':distribution:tools:upgrade-cli') bcFips libs.bundles.bouncycastle } @@ -356,9 +355,6 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { into('tools/keystore-cli') { from(configurations.libsKeystoreCli) } - into('tools/upgrade-cli') { - from(configurations.libsUpgradeCli) - } } } diff --git a/distribution/src/bin/opensearch-upgrade b/distribution/src/bin/opensearch-upgrade deleted file mode 100755 index 5aae184341b6b..0000000000000 --- a/distribution/src/bin/opensearch-upgrade +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -e -o pipefail - -OPENSEARCH_MAIN_CLASS=org.opensearch.upgrade.UpgradeCli \ - OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/upgrade-cli \ - "`dirname "$0"`"/opensearch-cli \ - "$@" diff --git a/distribution/src/bin/opensearch-upgrade.bat b/distribution/src/bin/opensearch-upgrade.bat deleted file mode 100644 index f44b3750eb9e0..0000000000000 --- a/distribution/src/bin/opensearch-upgrade.bat +++ /dev/null @@ -1,16 +0,0 @@ -@echo off - -setlocal enabledelayedexpansion -setlocal enableextensions - -set OPENSEARCH_MAIN_CLASS=org.opensearch.upgrade.UpgradeCli -set OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/upgrade-cli -call "%~dp0opensearch-cli.bat" ^ - %%* ^ - || goto exit - - -endlocal -endlocal -:exit -exit /b %ERRORLEVEL% diff --git a/distribution/tools/upgrade-cli/build.gradle b/distribution/tools/upgrade-cli/build.gradle deleted file mode 100644 index 92c043132c021..0000000000000 --- a/distribution/tools/upgrade-cli/build.gradle +++ /dev/null @@ -1,35 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -apply plugin: 'opensearch.build' - -base { - archivesName = 'opensearch-upgrade-cli' -} - -dependencies { - compileOnly project(":server") - compileOnly project(":libs:opensearch-cli") - implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" - implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" - implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" - testImplementation project(":test:framework") - testImplementation 'com.google.jimfs:jimfs:1.3.0' - testRuntimeOnly("com.google.guava:guava:${versions.guava}") { - transitive = false - } -} - -tasks.named("dependencyLicenses").configure { - mapping from: /jackson-.*/, to: 'jackson' -} - -test { - systemProperty 'tests.security.manager', 'false' -} diff --git a/distribution/tools/upgrade-cli/licenses/jackson-LICENSE b/distribution/tools/upgrade-cli/licenses/jackson-LICENSE deleted file mode 100644 index f5f45d26a49d6..0000000000000 --- a/distribution/tools/upgrade-cli/licenses/jackson-LICENSE +++ /dev/null @@ -1,8 +0,0 @@ -This copy of Jackson JSON processor streaming parser/generator is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 diff --git a/distribution/tools/upgrade-cli/licenses/jackson-NOTICE b/distribution/tools/upgrade-cli/licenses/jackson-NOTICE deleted file mode 100644 index 4c976b7b4cc58..0000000000000 --- a/distribution/tools/upgrade-cli/licenses/jackson-NOTICE +++ /dev/null @@ -1,20 +0,0 @@ -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -## Licensing - -Jackson core and extension components may licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -## Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. diff --git a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.18.2.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.18.2.jar.sha1 deleted file mode 100644 index a06e1d5f28425..0000000000000 --- a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.18.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -985d77751ebc7fce5db115a986bc9aa82f973f4a \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.18.2.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.18.2.jar.sha1 deleted file mode 100644 index eedbfff66c705..0000000000000 --- a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.18.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -deef8697b92141fb6caf7aa86966cff4eec9b04f \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/DetectEsInstallationTask.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/DetectEsInstallationTask.java deleted file mode 100644 index de2d24dc61ce1..0000000000000 --- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/DetectEsInstallationTask.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import org.opensearch.Version; -import org.opensearch.cli.Terminal; -import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.collect.Tuple; -import org.opensearch.common.settings.Settings; - -import java.io.File; -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Scanner; - -/** - * Looks for an existing elasticsearch installation. First it tries to identify automatically, - * and if unsuccessful, asks the user to input the missing details. - *

    - * If an elasticsearch installation can not be found, throws a runtime error which fails the - * upgrade task. - */ -class DetectEsInstallationTask implements UpgradeTask { - private static final int ES_DEFAULT_PORT = 9200; - private static final String ES_CONFIG_ENV = "ES_PATH_CONF"; - private static final String ES_CONFIG_YML = "elasticsearch.yml"; - private static final String ES_HOME = "ES_HOME"; - - @SuppressForbidden(reason = "We need to read external es config files") - @Override - public void accept(final Tuple input) { - final TaskInput taskInput = input.v1(); - final Terminal terminal = input.v2(); - try { - terminal.println("Looking for an elasticsearch installation ..."); - String esHomeEnv = System.getenv(ES_HOME); - if (esHomeEnv == null) { - esHomeEnv = terminal.readText("Missing ES_HOME env variable, enter the path to elasticsearch home: "); - if (esHomeEnv == null || esHomeEnv.isEmpty()) { - throw new RuntimeException("Invalid input for path to elasticsearch home directory."); - } - } - taskInput.setEsHome(new File(esHomeEnv).toPath()); - - String esConfEnv = System.getenv(ES_CONFIG_ENV); - if (esConfEnv == null) { - esConfEnv = terminal.readText("Missing ES_PATH_CONF env variable, enter the path to elasticsearch config directory: "); - if (esConfEnv == null || esHomeEnv.isEmpty()) { - throw new RuntimeException("Invalid input for path to elasticsearch config directory."); - } - } - taskInput.setEsConfig(new File(esConfEnv).toPath()); - - final Settings esSettings = Settings.builder().loadFromPath(taskInput.getEsConfig().resolve(ES_CONFIG_YML)).build(); - final String url = retrieveUrl(esSettings); - taskInput.setBaseUrl(url); - final boolean running = isRunning(url); - taskInput.setRunning(running); - if (running) { - terminal.println("Found a running instance of elasticsearch at " + url); - taskInput.setRunning(true); - try { - updateTaskInput(taskInput, fetchInfoFromUrl(taskInput.getBaseUrl())); - } catch (RuntimeException e) { - updateTaskInput(taskInput, fetchInfoFromEsSettings(esSettings)); - } - try { - taskInput.setPlugins(fetchPluginsFromUrl(taskInput.getBaseUrl())); - } catch (RuntimeException e) { - taskInput.setPlugins(detectPluginsFromEsHome(taskInput.getEsHome())); - } - } else { - terminal.println("Did not find a running instance of elasticsearch at " + url); - updateTaskInput(taskInput, fetchInfoFromEsSettings(esSettings)); - taskInput.setPlugins(detectPluginsFromEsHome(taskInput.getEsHome())); - } - } catch (IOException e) { - throw new RuntimeException("Error detecting existing elasticsearch installation. " + e); - } - } - - @SuppressWarnings("unchecked") - private void updateTaskInput(TaskInput taskInput, Map response) { - final Map versionMap = (Map) response.get("version"); - if (versionMap != null) { - final String vStr = versionMap.get("number"); - if (vStr != null) { - taskInput.setVersion(Version.fromString(vStr)); - } - } - taskInput.setNode((String) response.get("name")); - taskInput.setCluster((String) response.get("cluster_name")); - } - - // package private for unit testing - String retrieveUrl(final Settings esSettings) { - final int port = Optional.ofNullable(esSettings.get("http.port")).map(this::extractPort).orElse(ES_DEFAULT_PORT); - return "http://localhost:" + port; - } - - private Integer extractPort(final String port) { - try { - return Integer.parseInt(port.trim()); - } catch (Exception ex) { - return ES_DEFAULT_PORT; - } - } - - @SuppressForbidden(reason = "Need to connect to http endpoint for elasticsearch.") - private boolean isRunning(final String url) { - try { - final URL esUrl = new URL(url); - final HttpURLConnection conn = (HttpURLConnection) esUrl.openConnection(); - conn.setRequestMethod("GET"); - conn.setConnectTimeout(1000); - conn.connect(); - return conn.getResponseCode() == 200; - } catch (IOException e) { - return false; - } - } - - @SuppressForbidden(reason = "Retrieve information on the installation.") - private Map fetchInfoFromUrl(final String url) { - try { - final URL esUrl = new URL(url); - final HttpURLConnection conn = (HttpURLConnection) esUrl.openConnection(); - conn.setRequestMethod("GET"); - conn.setConnectTimeout(1000); - conn.connect(); - - final StringBuilder json = new StringBuilder(); - final Scanner scanner = new Scanner(esUrl.openStream()); - while (scanner.hasNext()) { - json.append(scanner.nextLine()); - } - scanner.close(); - final ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(json.toString(), Map.class); - } catch (IOException e) { - throw new RuntimeException("Error retrieving elasticsearch cluster info, " + e); - } - } - - private Map fetchInfoFromEsSettings(final Settings esSettings) throws IOException { - final Map info = new HashMap<>(); - final String node = esSettings.get("node.name") != null ? esSettings.get("node.name") : "unknown"; - final String cluster = esSettings.get("cluster.name") != null ? esSettings.get("cluster.name") : "unknown"; - info.put("name", node); - info.put("cluster_name", cluster); - return info; - } - - @SuppressWarnings("unchecked") - @SuppressForbidden(reason = "Retrieve information on installed plugins.") - private List fetchPluginsFromUrl(final String url) { - final List plugins = new ArrayList<>(); - try { - final URL esUrl = new URL(url + "/_cat/plugins?format=json&local=true"); - final HttpURLConnection conn = (HttpURLConnection) esUrl.openConnection(); - conn.setRequestMethod("GET"); - conn.setConnectTimeout(1000); - conn.connect(); - if (conn.getResponseCode() == 200) { - final StringBuilder json = new StringBuilder(); - final Scanner scanner = new Scanner(esUrl.openStream()); - while (scanner.hasNext()) { - json.append(scanner.nextLine()); - } - scanner.close(); - final ObjectMapper mapper = new ObjectMapper(); - final Map[] response = mapper.readValue(json.toString(), Map[].class); - for (Map plugin : response) { - plugins.add(plugin.get("component")); - } - } - return plugins; - } catch (IOException e) { - throw new RuntimeException("Error retrieving elasticsearch plugin details, " + e); - } - } - - private List detectPluginsFromEsHome(final Path esHome) { - // list out the contents of the plugins directory under esHome - return Collections.emptyList(); - } -} diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportJvmOptionsTask.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportJvmOptionsTask.java deleted file mode 100644 index fcb23f02425c6..0000000000000 --- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportJvmOptionsTask.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import org.opensearch.cli.Terminal; -import org.opensearch.common.collect.Tuple; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Imports JVM options from an existing elasticsearch installation. - */ -class ImportJvmOptionsTask implements UpgradeTask { - private static final String JVM_OPTIONS_D = "jvm.options.d"; - - @Override - public void accept(final Tuple input) { - final TaskInput taskInput = input.v1(); - final Terminal terminal = input.v2(); - try { - terminal.println("Importing JVM options ..."); - final Path jvmOptionsDir = taskInput.getOpenSearchConfig().resolve(JVM_OPTIONS_D); - if (!Files.exists(jvmOptionsDir)) { - Files.createDirectory(jvmOptionsDir); - } - - final Path esJvmOptionsDir = taskInput.getEsConfig().resolve(JVM_OPTIONS_D); - if (Files.exists(esJvmOptionsDir) && Files.isDirectory(esJvmOptionsDir)) { - final List esJvmOptionsFiles = Files.list(esJvmOptionsDir).collect(Collectors.toList()); - for (Path esJvmOptFile : esJvmOptionsFiles) { - final Path jvmOptFile = jvmOptionsDir.resolve(esJvmOptFile.getFileName().toString()); - Files.copy(esJvmOptFile, jvmOptFile, StandardCopyOption.REPLACE_EXISTING); - } - } - terminal.println("Success!" + System.lineSeparator()); - } catch (Exception e) { - throw new RuntimeException("Error importing JVM options. " + e); - } - } -} diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportKeystoreTask.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportKeystoreTask.java deleted file mode 100644 index f2f36ab3bbe0e..0000000000000 --- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportKeystoreTask.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import org.opensearch.cli.Terminal; -import org.opensearch.common.collect.Tuple; -import org.opensearch.common.settings.KeyStoreWrapper; -import org.opensearch.core.common.settings.SecureString; - -import java.io.InputStream; - -/** - * Imports the secure Keystore settings from an existing elasticsearch installation. - */ -class ImportKeystoreTask implements UpgradeTask { - private static final String OPENSEARCH_KEYSTORE_FILENAME = "opensearch.keystore"; - private static final String ES_KEYSTORE_FILENAME = "elasticsearch.keystore"; - - @Override - public void accept(final Tuple input) { - final TaskInput taskInput = input.v1(); - final Terminal terminal = input.v2(); - SecureString keyStorePassword = new SecureString(new char[0]); - try { - terminal.println("Importing keystore settings ..."); - final KeyStoreWrapper esKeystore = KeyStoreWrapper.load(taskInput.getEsConfig(), ES_KEYSTORE_FILENAME); - if (esKeystore == null) { - terminal.println("No elasticsearch keystore settings to import."); - return; - } - KeyStoreWrapper openSearchKeystore = KeyStoreWrapper.load( - taskInput.getOpenSearchConfig().resolve(OPENSEARCH_KEYSTORE_FILENAME) - ); - if (openSearchKeystore == null) { - openSearchKeystore = KeyStoreWrapper.create(); - } - if (esKeystore.hasPassword()) { - final char[] passwordArray = terminal.readSecret("Enter password for the elasticsearch keystore : "); - keyStorePassword = new SecureString(passwordArray); - } - esKeystore.decrypt(keyStorePassword.getChars()); - for (String setting : esKeystore.getSettingNames()) { - if (setting.equals("keystore.seed")) { - continue; - } - if (!openSearchKeystore.getSettingNames().contains(setting)) { - InputStream settingIS = esKeystore.getFile(setting); - byte[] bytes = new byte[settingIS.available()]; - settingIS.read(bytes); - KeystoreWrapperUtil.saveSetting(openSearchKeystore, setting, bytes); - } - } - openSearchKeystore.save(taskInput.getOpenSearchConfig(), keyStorePassword.getChars()); - terminal.println("Success!" + System.lineSeparator()); - } catch (Exception e) { - throw new RuntimeException("Error importing keystore settings from elasticsearch, " + e); - } finally { - keyStorePassword.close(); - } - } -} diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportLog4jPropertiesTask.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportLog4jPropertiesTask.java deleted file mode 100644 index da999a3f91f08..0000000000000 --- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportLog4jPropertiesTask.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import org.opensearch.cli.Terminal; -import org.opensearch.common.collect.Tuple; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; -import java.util.Map; -import java.util.Properties; - -/** - * Imports Log4j properties from an existing elasticsearch installation. - */ -class ImportLog4jPropertiesTask implements UpgradeTask { - static final String LOG4J_PROPERTIES = "log4j2.properties"; - - @Override - public void accept(final Tuple input) { - final TaskInput taskInput = input.v1(); - final Terminal terminal = input.v2(); - try { - terminal.println("Importing log4j.properties ..."); - final Path log4jPropPath = taskInput.getOpenSearchConfig().resolve(LOG4J_PROPERTIES); - if (Files.exists(log4jPropPath)) { - Files.copy( - log4jPropPath, - taskInput.getOpenSearchConfig().resolve(LOG4J_PROPERTIES + ".bkp"), - StandardCopyOption.REPLACE_EXISTING - ); - } - final Path esLog4jPropPath = taskInput.getEsConfig().resolve(LOG4J_PROPERTIES); - try ( - InputStream esLog4jIs = Files.newInputStream(esLog4jPropPath); - OutputStream log4jOs = Files.newOutputStream(log4jPropPath, StandardOpenOption.TRUNCATE_EXISTING) - ) { - final Properties esLog4JProps = new Properties(); - esLog4JProps.load(esLog4jIs); - final Properties log4jProps = renameValues(esLog4JProps); - - log4jProps.store(log4jOs, "This is an auto-generated file imported from an existing elasticsearch installation."); - } - terminal.println("Success!" + System.lineSeparator()); - } catch (IOException e) { - throw new RuntimeException("Error copying log4j properties. " + e); - } - } - - /** - * Rename the values for OpenSearch log4j properties to reflect the changed names - * for java packages, class names and system variables. - * - * @param esLog4JProps existing elasticsearch log4j properties. - * @return updated properties for OpenSearch. - */ - private Properties renameValues(Properties esLog4JProps) { - final Properties props = new Properties(); - for (Map.Entry entry : esLog4JProps.entrySet()) { - final String key = (String) entry.getKey(); - final String value = (String) entry.getValue(); - final String newKey = key.replaceAll("esmessagefields", "opensearchmessagefields"); - final String newValue = value.replaceAll("ESJsonLayout", "OpenSearchJsonLayout") - .replaceAll("sys:es.logs", "sys:opensearch.logs") - .replaceAll("org.elasticsearch", "org.opensearch"); - props.setProperty(newKey, newValue); - } - return props; - } -} diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportYmlConfigTask.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportYmlConfigTask.java deleted file mode 100644 index 2ba8f93cd53ac..0000000000000 --- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ImportYmlConfigTask.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import org.opensearch.cli.Terminal; -import org.opensearch.common.collect.Tuple; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.settings.SettingsException; -import org.opensearch.common.xcontent.yaml.YamlXContent; -import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.core.xcontent.XContentBuilder; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** - * Imports settings from an existing elasticsearch installation. - */ -class ImportYmlConfigTask implements UpgradeTask { - private static final String ES_CONFIG_FILENAME = "elasticsearch.yml"; - private static final String OPENSEARCH_CONFIG_FILENAME = "opensearch.yml"; - static final String HEADER = "# ======================== OpenSearch Configuration =========================\n" - + "# NOTE: The settings in this file are imported from an existing Elasticsearch\n" - + "# installation using the opensearch-upgrade tool. The original file is\n" - + "# backed up in this directory as opensearch.yml.bkp for reference.\n\n" - + "# Please consult the documentation for further information:\n" - + "# https://www.opensearch.org\n" - + "#\n"; - - @Override - public void accept(final Tuple input) { - final TaskInput taskInput = input.v1(); - final Terminal terminal = input.v2(); - try { - terminal.println("Importing settings from elasticsearch.yml ..."); - final Path openSearchYmlPath = taskInput.getOpenSearchConfig().resolve(OPENSEARCH_CONFIG_FILENAME); - final Path esYamlPath = taskInput.getEsConfig().resolve(ES_CONFIG_FILENAME); - final Settings esSettings = Settings.builder().loadFromPath(esYamlPath).build(); - final Settings settings = Settings.builder().loadFromPath(openSearchYmlPath).build(); - if (esSettings.size() > 0) { - if (settings.size() > 0 - && terminal.promptYesNo("Existing settings in opensearch.yml will be overwritten, proceed?", false) == false) { - terminal.println("Import settings cancelled by user"); - } - final Path backupYmlPath = taskInput.getOpenSearchConfig().resolve(OPENSEARCH_CONFIG_FILENAME + ".bkp"); - if (!Files.exists(backupYmlPath) - || terminal.promptYesNo("A backup file for opensearch.yml already exists, overwrite?", false)) { - Files.copy(openSearchYmlPath, backupYmlPath, StandardCopyOption.REPLACE_EXISTING); - } - Files.write(openSearchYmlPath, Collections.singleton(HEADER), StandardOpenOption.TRUNCATE_EXISTING); - final Settings mergeSettings = mergeSettings(settings, esSettings); - writeSettings(openSearchYmlPath, mergeSettings); - } - terminal.println("Success!" + System.lineSeparator()); - } catch (IOException ex) { - throw new RuntimeException("Error importing settings from elasticsearch.yml, " + ex); - } - } - - // package private for unit testing - Settings mergeSettings(final Settings first, final Settings second) { - Settings.Builder builder = Settings.builder(); - for (String key : first.keySet()) { - builder.copy(key, key, first); - } - for (String key : second.keySet()) { - builder.copy(key, key, second); - } - return builder.build(); - } - - /** - * Write settings to the config file on the file system. It uses the {@link XContentBuilder} - * to build the YAML content and write it to the output stream. - * - * @param configYml path to a yml file where config will be written to. - * @param settings the settings to write - * @throws IOException exception during writing to the output stream. - */ - private void writeSettings(final Path configYml, final Settings settings) throws IOException { - try ( - OutputStream os = Files.newOutputStream(configYml, StandardOpenOption.APPEND); - XContentBuilder builder = new XContentBuilder(YamlXContent.yamlXContent, os) - ) { - builder.startObject(); - final Map params = new HashMap<>(); - params.put("flat_settings", "true"); - settings.toXContent(builder, new ToXContent.MapParams(params)); - builder.endObject(); - builder.flush(); - } catch (Exception e) { - throw new SettingsException("Failed to write settings to " + configYml.toString(), e); - } - } -} diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/InstallPluginsTask.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/InstallPluginsTask.java deleted file mode 100644 index ae2335752bed0..0000000000000 --- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/InstallPluginsTask.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import org.opensearch.cli.Terminal; -import org.opensearch.common.collect.Tuple; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Set; - -/** - * Installs the list of plugins using the opensearch-plugin command. -*/ -class InstallPluginsTask implements UpgradeTask { - private static final String ERROR_MSG = "Error installing plugin %s. Please install it manually."; - - /** The list of official plugins that can be installed by the upgrade tool. */ - static final Set OFFICIAL_PLUGINS; - static { - try ( - InputStream stream = InstallPluginsTask.class.getResourceAsStream("/plugins.txt"); - BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)) - ) { - Set plugins = new HashSet<>(); - String line = reader.readLine(); - while (line != null) { - plugins.add(line.trim()); - line = reader.readLine(); - } - OFFICIAL_PLUGINS = Collections.unmodifiableSet(plugins); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public void accept(final Tuple input) { - final TaskInput taskInput = input.v1(); - final Terminal terminal = input.v2(); - if (taskInput.getPlugins() == null || taskInput.getPlugins().isEmpty()) { - return; - } - terminal.println("Installing core plugins ..."); - List manualPlugins = new ArrayList<>(); - - for (String plugin : taskInput.getPlugins()) { - if (OFFICIAL_PLUGINS.contains(plugin)) { - executeInstallPluginCommand(plugin, taskInput, terminal); - } else { - manualPlugins.add(plugin); - } - } - if (!manualPlugins.isEmpty()) { - terminal.println("Please install the following custom plugins manually: " + manualPlugins); - } - terminal.println("Success!" + System.lineSeparator()); - } - - // package private for unit testing - void executeInstallPluginCommand(String plugin, TaskInput taskInput, Terminal terminal) { - ProcessBuilder processBuilder = getProcessBuilderBasedOnOS(plugin, taskInput); - try { - final Process process = processBuilder.inheritIO().start(); - if (process.waitFor() != 0) { - terminal.errorPrint(Terminal.Verbosity.NORMAL, String.format(Locale.getDefault(), ERROR_MSG, plugin)); - } - } catch (IOException | InterruptedException e) { - terminal.errorPrint(Terminal.Verbosity.NORMAL, String.format(Locale.getDefault(), ERROR_MSG, plugin) + e.getMessage()); - } - } - - // package private for unit testing - ProcessBuilder getProcessBuilderBasedOnOS(String plugin, TaskInput taskInput) { - final String command = taskInput.getOpenSearchBin().resolve("opensearch-plugin") + " install " + plugin; - final ProcessBuilder processBuilder = new ProcessBuilder(); - if (OS.WINDOWS == OS.current()) { - processBuilder.command("cmd.exe", "/c", command); - } else { - processBuilder.command("sh", "-c", command); - } - return processBuilder; - } - - private enum OS { - WINDOWS, - MAC, - LINUX; - - public static OS current() { - final String os = System.getProperty("os.name", ""); - if (os.startsWith("Windows")) { - return OS.WINDOWS; - } - if (os.startsWith("Linux") || os.startsWith("LINUX")) { - return OS.LINUX; - } - if (os.startsWith("Mac")) { - return OS.MAC; - } - throw new IllegalStateException("Can't determine OS from: " + os); - } - } -} diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/KeystoreWrapperUtil.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/KeystoreWrapperUtil.java deleted file mode 100644 index 3ef7d09edc046..0000000000000 --- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/KeystoreWrapperUtil.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import org.opensearch.common.settings.KeyStoreWrapper; - -/** - * Utility that has package level access to the {@link KeyStoreWrapper} for - * saving a setting. - */ -public final class KeystoreWrapperUtil { - /** - * No public constructor. Contains only static functions. - */ - private KeystoreWrapperUtil() {} - - /** - * Save a secure setting using the wrapper. - * - * @param keystore an instance of {@link KeyStoreWrapper} - * @param setting setting to save - * @param bytes value of the setting in bytes - */ - public static void saveSetting(KeyStoreWrapper keystore, String setting, byte[] bytes) { - keystore.setFile(setting, bytes); - } -} diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/TaskInput.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/TaskInput.java deleted file mode 100644 index b3c5604275b54..0000000000000 --- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/TaskInput.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import org.opensearch.Version; -import org.opensearch.env.Environment; - -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; - -/** - * A plain old java object, that contains the information used by tasks - * in the upgrade process. - */ -class TaskInput { - private final Environment openSearchEnv; - private String node; - private String cluster; - private String baseUrl; - private boolean running; - private Version version; - private List plugins; - private Path esHome; - private Path esConfig; - - TaskInput(Environment openSearchEnv) { - this.openSearchEnv = openSearchEnv; - } - - public String getNode() { - return node; - } - - public void setNode(String node) { - this.node = node; - } - - public String getCluster() { - return cluster; - } - - public void setCluster(String cluster) { - this.cluster = cluster; - } - - public Optional getVersion() { - return Optional.ofNullable(version); - } - - public void setVersion(Version version) { - this.version = version; - } - - public List getPlugins() { - return plugins; - } - - public void setPlugins(List plugins) { - this.plugins = plugins; - } - - public Path getEsConfig() { - return esConfig; - } - - public void setEsConfig(Path esConfig) { - this.esConfig = esConfig; - } - - public Path getEsHome() { - return esHome; - } - - public void setEsHome(Path esHome) { - this.esHome = esHome; - } - - public Path getOpenSearchConfig() { - return openSearchEnv.configDir(); - } - - public Path getOpenSearchBin() { - return openSearchEnv.binDir(); - } - - public boolean isRunning() { - return running; - } - - public void setRunning(boolean running) { - this.running = running; - } - - public String getBaseUrl() { - return baseUrl; - } - - public void setBaseUrl(String baseUrl) { - this.baseUrl = baseUrl; - } -} diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/UpgradeCli.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/UpgradeCli.java deleted file mode 100644 index f609d06b8ed34..0000000000000 --- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/UpgradeCli.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import joptsimple.OptionSet; -import org.opensearch.cli.ExitCodes; -import org.opensearch.cli.Terminal; -import org.opensearch.cli.UserException; -import org.opensearch.common.cli.EnvironmentAwareCommand; -import org.opensearch.common.collect.Tuple; -import org.opensearch.env.Environment; - -/** - * This class extends the existing opensearch-cli and provides the entry - * point for the opensearch-upgrade tool. - *

    - * This class is agnostic of the actual logic which performs the upgrade - * on the node. - */ -public class UpgradeCli extends EnvironmentAwareCommand { - - /** - * Constructor to create an instance of UpgradeCli. - */ - public UpgradeCli() { - super("A CLI tool for upgrading to OpenSearch 1.x from a supported Elasticsearch version."); - } - - /** - * Entrypoint for the upgrade tool. - * - * @param args args to main. - * @throws Exception exception thrown during the execution of the UpgradeCli. - */ - public static void main(String[] args) throws Exception { - exit(new UpgradeCli().main(args, Terminal.DEFAULT)); - } - - /** - * Executes the upgrade task. This retrieves an instance of {@link UpgradeTask} which is composed - * of smaller individual tasks that perform specific operations as part of the overall process. - * - * @param terminal current terminal the command is running - * @param options options supplied to the command - * @param env current environment in which this cli tool is running. - * @throws UserException if any exception is thrown from the tasks - */ - @Override - protected void execute(final Terminal terminal, final OptionSet options, final Environment env) throws UserException { - try { - final Tuple input = new Tuple<>(new TaskInput(env), terminal); - UpgradeTask.getTask().accept(input); - terminal.println("Done!"); - terminal.println("Next Steps: "); - terminal.println(" Stop the running elasticsearch on this node."); - terminal.println(" Start OpenSearch on this node."); - } catch (RuntimeException ex) { - throw new UserException(ExitCodes.DATA_ERROR, ex.getMessage()); - } - } -} diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/UpgradeTask.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/UpgradeTask.java deleted file mode 100644 index 8f84dd8c9817c..0000000000000 --- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/UpgradeTask.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import org.opensearch.cli.Terminal; -import org.opensearch.common.collect.Tuple; - -import java.util.function.Consumer; - -/** - * An interface for an upgrade task, which in this instance is an unit of - * operation that is part of the overall upgrade process. This extends the - * {@link java.util.function.Consumer} interface. - *

    - * The implementing tasks consume and instance of a tuple of {@link TaskInput} - * and {@link Terminal} and operate via side effects. - * - */ -interface UpgradeTask extends Consumer> { - /** - * Composes the individual tasks to create a pipeline for the overall upgrade task. - * - * @return an instance of {@link java.util.function.Consumer} that takes a tuple of - * task input and the current terminal. The composed task fails if any of the - * individual tasks fails. - */ - static Consumer> getTask() { - return new DetectEsInstallationTask().andThen(new ValidateInputTask()) - .andThen(new ImportYmlConfigTask()) - .andThen(new ImportJvmOptionsTask()) - .andThen(new ImportLog4jPropertiesTask()) - .andThen(new InstallPluginsTask()) - .andThen(new ImportKeystoreTask()); - } -} diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ValidateInputTask.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ValidateInputTask.java deleted file mode 100644 index bea524e651827..0000000000000 --- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/ValidateInputTask.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import org.opensearch.LegacyESVersion; -import org.opensearch.Version; -import org.opensearch.cli.Terminal; -import org.opensearch.common.collect.Tuple; - -import java.util.LinkedHashMap; -import java.util.Locale; -import java.util.Map; - -/** - * Validates the input {@link TaskInput} for the upgrade. - */ -class ValidateInputTask implements UpgradeTask { - - @Override - public void accept(final Tuple input) { - final TaskInput taskInput = input.v1(); - final Terminal terminal = input.v2(); - - terminal.println("Verifying the details ..."); - // check if the elasticsearch version is supported - if (taskInput.getVersion().isPresent()) { - final Version version = taskInput.getVersion().get(); - if (version.equals(LegacyESVersion.fromId(7100299)) == false) { - throw new RuntimeException( - String.format(Locale.getDefault(), "The installed version %s of elasticsearch is not supported.", version) - ); - } - } else { - terminal.println("Unable to detect installed elasticsearch version."); - confirmToProceed(terminal); - } - // check if the OpenSearch config is set to an external location - if (taskInput.getOpenSearchConfig().getParent().equals(taskInput.getOpenSearchBin().getParent())) { - terminal.println( - "OpenSearch config directory is set inside the installation directory. " - + "It is recommended to use an external config directory and set the environment variable " - + "OPENSEARCH_PATH_CONF to it." - ); - confirmToProceed(terminal); - } - - // print summary and confirm with user if everything looks correct. - final Map fieldsMap = getSummaryFieldsMap(taskInput); - final String format = " %-25s | %s"; - terminal.println("+----------------------- SUMMARY -----------------------+"); - for (Map.Entry entry : fieldsMap.entrySet()) { - terminal.println(String.format(Locale.getDefault(), format, entry.getKey(), entry.getValue())); - } - terminal.println("+-------------------------------------------------------+"); - terminal.println("Please verify if everything above looks good."); - confirmToProceed(terminal); - } - - private void confirmToProceed(final Terminal terminal) { - terminal.println(System.lineSeparator()); - if (terminal.promptYesNo("Do you want to proceed?", false) == false) { - throw new RuntimeException("Upgrade cancelled by user."); - } - } - - // package private for unit testing - Map getSummaryFieldsMap(final TaskInput taskInput) { - final String version = taskInput.getVersion().isPresent() ? taskInput.getVersion().get().toString() : "unknown"; - - final Map fields = new LinkedHashMap<>(); - fields.put("Cluster", taskInput.getCluster()); - fields.put("Node", taskInput.getNode()); - fields.put("Endpoint", taskInput.getBaseUrl()); - fields.put("Elasticsearch Version", version); - fields.put("Elasticsearch Config", taskInput.getEsConfig().toString()); - fields.put("Elasticsearch Plugins", taskInput.getPlugins() == null ? "[]" : taskInput.getPlugins().toString()); - fields.put("OpenSearch Config", taskInput.getOpenSearchConfig().toString()); - - return fields; - } -} diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/package-info.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/package-info.java deleted file mode 100644 index 86ecef5ae5576..0000000000000 --- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/tools/cli/upgrade/package-info.java +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** - * This exists to get access to the package level methods of KeyStoreWrapper. - */ -package org.opensearch.tools.cli.upgrade; diff --git a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/DetectEsInstallationTaskTests.java b/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/DetectEsInstallationTaskTests.java deleted file mode 100644 index e653b3fa8d284..0000000000000 --- a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/DetectEsInstallationTaskTests.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import org.opensearch.cli.MockTerminal; -import org.opensearch.cli.Terminal; -import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.collect.Tuple; -import org.opensearch.common.settings.Settings; -import org.opensearch.env.Environment; -import org.opensearch.env.TestEnvironment; -import org.opensearch.test.OpenSearchTestCase; -import org.junit.Before; - -import java.io.File; -import java.nio.file.Path; - -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; - -public class DetectEsInstallationTaskTests extends OpenSearchTestCase { - - private final MockTerminal terminal = new MockTerminal(); - private DetectEsInstallationTask task; - private Environment env; - - @Before - public void setUpTask() { - task = new DetectEsInstallationTask(); - env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", "").build()); - } - - @SuppressForbidden(reason = "Read config directory from test resources.") - public void testTaskExecution() throws Exception { - Path esConfig = new File(getClass().getResource("/config").getPath()).toPath(); - // path for es_home - terminal.addTextInput(esConfig.getParent().toString()); - // path for es_config - terminal.addTextInput(esConfig.toString()); - TaskInput taskInput = new TaskInput(env); - Tuple input = new Tuple<>(taskInput, terminal); - - task.accept(input); - - assertThat(taskInput.getEsConfig(), is(esConfig)); - assertThat(taskInput.getBaseUrl(), is("http://localhost:42123")); - assertThat(taskInput.getPlugins(), hasSize(0)); - assertThat(taskInput.getNode(), is("node-x")); - assertThat(taskInput.getCluster(), is("my-cluster")); - } - - public void testRetrieveUrlFromSettings() { - Settings esSettings = Settings.builder().put("http.port", "9201").build(); - - assertThat(task.retrieveUrl(esSettings), is("http://localhost:9201")); - } - - public void testRetrieveDefaultUrlFromConfig() { - assertThat(task.retrieveUrl(Settings.EMPTY), is("http://localhost:9200")); - } -} diff --git a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/ImportLog4jPropertiesTaskTests.java b/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/ImportLog4jPropertiesTaskTests.java deleted file mode 100644 index 8136504b15462..0000000000000 --- a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/ImportLog4jPropertiesTaskTests.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import com.google.common.jimfs.Configuration; -import com.google.common.jimfs.Jimfs; -import org.opensearch.cli.MockTerminal; -import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.PathUtilsForTesting; -import org.opensearch.common.settings.Settings; -import org.opensearch.env.Environment; -import org.opensearch.env.TestEnvironment; -import org.opensearch.test.OpenSearchTestCase; -import org.junit.Before; - -import java.io.File; -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; - -public class ImportLog4jPropertiesTaskTests extends OpenSearchTestCase { - private final MockTerminal terminal = new MockTerminal(); - private final List fileSystems = new ArrayList<>(); - private ImportLog4jPropertiesTask task; - private Environment env; - - @Before - public void setUpTask() throws IOException { - task = new ImportLog4jPropertiesTask(); - final Configuration configuration; - configuration = Configuration.unix().toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build(); - FileSystem fs = Jimfs.newFileSystem(configuration); - fileSystems.add(fs); - PathUtilsForTesting.installMock(fs); - Path home = fs.getPath("test-home"); - Path config = home.resolve("config"); - Files.createDirectories(config); - Files.createFile(config.resolve(ImportLog4jPropertiesTask.LOG4J_PROPERTIES)); - env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", home).build()); - } - - @SuppressForbidden(reason = "Read config directory from test resources.") - public void testImportLog4jPropertiesTask() throws IOException { - TaskInput taskInput = new TaskInput(env); - Path esConfig = new File(getClass().getResource("/config").getPath()).toPath(); - taskInput.setEsConfig(esConfig); - task.accept(new Tuple<>(taskInput, terminal)); - - Properties properties = new Properties(); - properties.load(Files.newInputStream(taskInput.getOpenSearchConfig().resolve(ImportLog4jPropertiesTask.LOG4J_PROPERTIES))); - assertThat(properties, is(notNullValue())); - assertThat(properties.entrySet(), hasSize(165)); - assertThat(properties.get("appender.rolling.layout.type"), equalTo("OpenSearchJsonLayout")); - assertThat( - properties.get("appender.deprecation_rolling.fileName"), - equalTo("${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_deprecation.json") - ); - assertThat(properties.get("logger.deprecation.name"), equalTo("org.opensearch.deprecation")); - assertThat(properties.keySet(), not(hasItem("appender.deprecation_rolling.layout.esmessagefields"))); - assertThat(properties.keySet(), hasItem("appender.deprecation_rolling.layout.opensearchmessagefields")); - } -} diff --git a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/ImportYmlConfigTaskTests.java b/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/ImportYmlConfigTaskTests.java deleted file mode 100644 index 38dfebce7f21b..0000000000000 --- a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/ImportYmlConfigTaskTests.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import com.google.common.jimfs.Configuration; -import com.google.common.jimfs.Jimfs; -import org.opensearch.cli.MockTerminal; -import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.PathUtilsForTesting; -import org.opensearch.common.settings.Settings; -import org.opensearch.env.Environment; -import org.opensearch.env.TestEnvironment; -import org.opensearch.test.OpenSearchTestCase; -import org.junit.Before; - -import java.io.File; -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; - -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.is; - -public class ImportYmlConfigTaskTests extends OpenSearchTestCase { - private final MockTerminal terminal = new MockTerminal(); - private final List fileSystems = new ArrayList<>(); - private ImportYmlConfigTask task; - private Environment env; - - @Before - public void setUpTask() throws IOException { - task = new ImportYmlConfigTask(); - final Configuration configuration; - configuration = Configuration.unix().toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build(); - FileSystem fs = Jimfs.newFileSystem(configuration); - fileSystems.add(fs); - PathUtilsForTesting.installMock(fs); - Path home = fs.getPath("test-home"); - Path config = home.resolve("config"); - Files.createDirectories(config); - Files.createFile(config.resolve("opensearch.yml")); - env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", home).build()); - } - - @SuppressForbidden(reason = "Read config directory from test resources.") - public void testImportYmlConfigTask() throws IOException { - TaskInput taskInput = new TaskInput(env); - Path esConfig = new File(getClass().getResource("/config").getPath()).toPath(); - taskInput.setEsConfig(esConfig); - task.accept(new Tuple<>(taskInput, terminal)); - Settings settings = Settings.builder().loadFromPath(taskInput.getOpenSearchConfig().resolve("opensearch.yml")).build(); - assertThat(settings.keySet(), contains("cluster.name", "http.port", "node.name", "path.data", "path.logs")); - assertThat(settings.get("cluster.name"), is("my-cluster")); - assertThat(settings.get("http.port"), is("42123")); - assertThat(settings.get("node.name"), is("node-x")); - assertThat(settings.get("path.data"), is("[/mnt/data_1, /mnt/data_2]")); - assertThat(settings.get("path.logs"), is("/var/log/eslogs")); - } - - public void testMergeSettings() { - Settings first = Settings.builder().put("setting_one", "value_one").build(); - Settings second = Settings.builder().put("setting_two", "value_two").build(); - Settings merged = task.mergeSettings(first, second); - assertThat(merged.keySet(), contains("setting_one", "setting_two")); - assertThat(merged.get("setting_one"), is("value_one")); - assertThat(merged.get("setting_two"), is("value_two")); - } -} diff --git a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/InstallPluginsTaskTests.java b/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/InstallPluginsTaskTests.java deleted file mode 100644 index e7ac68331a8ca..0000000000000 --- a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/InstallPluginsTaskTests.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import org.opensearch.cli.MockTerminal; -import org.opensearch.common.collect.Tuple; -import org.opensearch.common.settings.Settings; -import org.opensearch.env.Environment; -import org.opensearch.env.TestEnvironment; -import org.opensearch.test.OpenSearchTestCase; -import org.junit.Before; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.mockito.Mockito; - -import static org.hamcrest.Matchers.containsString; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; - -public class InstallPluginsTaskTests extends OpenSearchTestCase { - - private final MockTerminal terminal = new MockTerminal(); - private InstallPluginsTask task; - private Environment env; - - private static final String OFFICIAL_PLUGIN = "analysis-icu"; - private static final String CUSTOM_PLUGIN = "job-scheduler"; - - @Before - public void setUpTask() throws IOException { - task = new InstallPluginsTask(); - env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", "").build()); - } - - public void testInstallPluginsTaskWithOfficialPlugin() throws IOException { - InstallPluginsTask spyTask = spy(task); - TaskInput taskInput = createTaskInputWithPlugin(OFFICIAL_PLUGIN); - spyTask.accept(new Tuple<>(taskInput, terminal)); - - verify(spyTask, Mockito.atLeast(1)).executeInstallPluginCommand(OFFICIAL_PLUGIN, taskInput, terminal); - } - - public void testInstallPluginsTaskWithCustomPlugin() throws IOException { - TaskInput taskInput = createTaskInputWithPlugin(CUSTOM_PLUGIN); - task.accept(new Tuple<>(taskInput, terminal)); - - assertThat(terminal.getOutput(), containsString("Please install the following custom plugins manually")); - } - - public void testGetCommandsBasedOnOS() { - TaskInput taskInput = createTaskInputWithPlugin(OFFICIAL_PLUGIN); - List commandsList = task.getProcessBuilderBasedOnOS(OFFICIAL_PLUGIN, taskInput).command(); - - final String os = System.getProperty("os.name", ""); - if (os.startsWith("Windows")) { - assertEquals("cmd.exe", commandsList.get(0)); - } else { - assertEquals("sh", commandsList.get(0)); - } - } - - private TaskInput createTaskInputWithPlugin(String plugin) { - TaskInput taskInput = new TaskInput(env); - List pluginsList = new ArrayList<>(); - pluginsList.add(plugin); - taskInput.setPlugins(pluginsList); - return taskInput; - } -} diff --git a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/UpgradeCliTests.java b/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/UpgradeCliTests.java deleted file mode 100644 index c1d1ca55ca315..0000000000000 --- a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/UpgradeCliTests.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import com.google.common.jimfs.Configuration; -import com.google.common.jimfs.Jimfs; -import org.opensearch.cli.Command; -import org.opensearch.cli.CommandTestCase; -import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.io.PathUtilsForTesting; -import org.opensearch.common.settings.KeyStoreWrapper; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.io.IOUtils; -import org.opensearch.env.Environment; -import org.opensearch.env.TestEnvironment; -import org.junit.After; -import org.junit.Before; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.GeneralSecurityException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasItems; -import static org.hamcrest.Matchers.is; - -public class UpgradeCliTests extends CommandTestCase { - private final List fileSystems = new ArrayList<>(); - private Environment env; - - @Before - public void setupEnv() throws IOException { - final Configuration configuration; - configuration = Configuration.unix().toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build(); - FileSystem fs = Jimfs.newFileSystem(configuration); - fileSystems.add(fs); - PathUtilsForTesting.installMock(fs); - Path home = fs.getPath("test-home"); - Path config = home.resolve("config"); - Files.createDirectories(config); - Files.createFile(config.resolve("opensearch.yml")); - Files.createDirectory(config.resolve("jvm.options.d")); - Files.createFile(config.resolve("log4j2.properties")); - env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", home).build()); - } - - @After - public void closeMockFileSystems() throws IOException { - IOUtils.close(fileSystems); - } - - @Override - protected Command newCommand() { - return new UpgradeCli() { - @Override - protected Environment createEnv(Map settings) { - return env; - } - }; - } - - @SuppressForbidden(reason = "Read config directory from test resources.") - public void testUpgrade() throws Exception { - String passwd = "keystorepassword"; - - Path esConfig = new File(getClass().getResource("/config").getPath()).toPath(); - // path for es_home - terminal.addTextInput(esConfig.getParent().toString()); - // path for es_config - terminal.addTextInput(esConfig.toString()); - // input for prompt 'config directory is inside installation' - terminal.addTextInput("y"); - // input for prompt 'es version not detected' - terminal.addTextInput("y"); - // input for prompt 'confirm the details look good' - terminal.addTextInput("y"); - // as the keystore is password protected, we set it. - terminal.addSecretInput(passwd); - - execute(); - - assertYmlConfigImported(); - assertKeystoreImported(passwd); - assertJvmOptionsImported(); - assertLog4jPropertiesImported(); - } - - private void assertYmlConfigImported() throws IOException { - String[] headers = ImportYmlConfigTask.HEADER.split("[\\r\\n]+"); - List expectedSettings = new ArrayList<>(); - expectedSettings.addAll(Arrays.asList(headers)); - // this is the generated flat settings - expectedSettings.addAll( - Arrays.asList( - "---", - "cluster.name: \"my-cluster\"", - "http.port: \"42123\"", - "node.name: \"node-x\"", - "path.data:", - "- \"/mnt/data_1\"", - "- \"/mnt/data_2\"", - "path.logs: \"/var/log/eslogs\"" - ) - ); - List actualSettings = Files.readAllLines(env.configDir().resolve("opensearch.yml")) - .stream() - .filter(Objects::nonNull) - .filter(line -> !line.isEmpty()) - .collect(Collectors.toList()); - - assertThat(actualSettings, equalTo(expectedSettings)); - } - - private void assertKeystoreImported(String passwd) throws IOException, GeneralSecurityException { - // assert keystore is created - KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configDir()); - assertNotNull(keystore); - - // assert all keystore settings are imported - keystore.decrypt(passwd.toCharArray()); - assertThat(keystore.getSettingNames(), hasItems(KeyStoreWrapper.SEED_SETTING.getKey(), "test.setting.key", "test.setting.file")); - assertThat(keystore.getString("test.setting.key").toString(), is("test.setting.value")); - InputStream is = keystore.getFile("test.setting.file"); - byte[] bytes = new byte[is.available()]; - assertThat(is.read(bytes), greaterThan(0)); - String actual = StandardCharsets.UTF_8.decode(ByteBuffer.wrap(bytes)).toString(); - String expected = "{\"some_key\": \"some_val\"}"; - assertThat(actual, is(expected)); - } - - private void assertJvmOptionsImported() throws IOException, GeneralSecurityException { - Path path = env.configDir().resolve("jvm.options.d"); - assertThat(Files.exists(path), is(true)); - assertThat(Files.isDirectory(path), is(true)); - assertThat(Files.exists(path.resolve("test.options")), is(true)); - } - - private void assertLog4jPropertiesImported() throws IOException, GeneralSecurityException { - assertThat(Files.exists(env.configDir().resolve("log4j2.properties")), is(true)); - } -} diff --git a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/ValidateInputTaskTests.java b/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/ValidateInputTaskTests.java deleted file mode 100644 index 91d708984deb4..0000000000000 --- a/distribution/tools/upgrade-cli/src/test/java/org/opensearch/tools/cli/upgrade/ValidateInputTaskTests.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.tools.cli.upgrade; - -import org.opensearch.LegacyESVersion; -import org.opensearch.cli.MockTerminal; -import org.opensearch.common.collect.Tuple; -import org.opensearch.common.io.PathUtils; -import org.opensearch.common.settings.Settings; -import org.opensearch.env.Environment; -import org.opensearch.env.TestEnvironment; -import org.opensearch.test.OpenSearchTestCase; -import org.junit.Before; - -import java.util.Arrays; -import java.util.Map; - -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; - -public class ValidateInputTaskTests extends OpenSearchTestCase { - - private ValidateInputTask task; - private MockTerminal terminal; - private Environment env; - - @Before - public void setTask() { - task = new ValidateInputTask(); - terminal = new MockTerminal(); - env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", "test_home").build()); - } - - public void testUnsupportedEsVersion() { - TaskInput taskInput = new TaskInput(env); - taskInput.setVersion(LegacyESVersion.fromId(7100199)); - - final RuntimeException e = expectThrows(RuntimeException.class, () -> task.accept(new Tuple<>(taskInput, terminal))); - - assertTrue(e.getMessage(), e.getMessage().contains("The installed version 7.10.1 of elasticsearch is not supported.")); - } - - public void testGetSummaryFields() { - TaskInput taskInput = new TaskInput(env); - taskInput.setEsConfig(PathUtils.get("es_home")); - taskInput.setCluster("some-cluster"); - taskInput.setNode("some-node"); - taskInput.setVersion(LegacyESVersion.fromId(7100299)); - taskInput.setBaseUrl("some-url"); - taskInput.setPlugins(Arrays.asList("plugin-1", "plugin-2")); - - Map summary = task.getSummaryFieldsMap(taskInput); - - assertThat(summary.entrySet(), hasSize(7)); - assertThat(summary.get("Cluster"), is("some-cluster")); - assertThat(summary.get("Node"), is("some-node")); - assertThat(summary.get("Endpoint"), is("some-url")); - assertThat(summary.get("Elasticsearch Version"), is("7.10.2")); - assertThat(summary.get("Elasticsearch Plugins"), is("[plugin-1, plugin-2]")); - assertThat(summary.get("Elasticsearch Config"), is("es_home")); - assertThat(summary.get("OpenSearch Config"), is(env.configDir().toString())); - } -} diff --git a/distribution/tools/upgrade-cli/src/test/resources/config/elasticsearch.keystore b/distribution/tools/upgrade-cli/src/test/resources/config/elasticsearch.keystore deleted file mode 100644 index 0d4d3468e5f1d17872bcbe11f36205d407b8c80b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 286 zcmV+(0pb2X*K8LSWo%({bZKLAWnpq-XfA7Id2@7ca%BJj00aR5008O$002Ou!oUS3 zF)Hoxxe?*J6(6x zp97seuIV`d000bg8@Nrb`S%Y)@3cYy005FRaLn7~M~yBzo`&TDl{Am!{?=C|eg_z1 z-~)_;;D377j;b8-JnX?U!5aF?x(WQc+nl!mmLUGY?1?=wI0bA7_a^Xt3B?3QU0v8n z;!tS^Jy#em;u1Ah#JgFfG0vfeM0>CJf=l}o!0000005Zb4bme$?od5s; diff --git a/distribution/tools/upgrade-cli/src/test/resources/config/elasticsearch.yml b/distribution/tools/upgrade-cli/src/test/resources/config/elasticsearch.yml deleted file mode 100644 index 083b4c7a7452b..0000000000000 --- a/distribution/tools/upgrade-cli/src/test/resources/config/elasticsearch.yml +++ /dev/null @@ -1,8 +0,0 @@ -cluster.name: my-cluster -node.name: node-x -path: - data: - - /mnt/data_1 - - /mnt/data_2 - logs: /var/log/eslogs -http.port: 42123 diff --git a/distribution/tools/upgrade-cli/src/test/resources/config/jvm.options.d/test.options b/distribution/tools/upgrade-cli/src/test/resources/config/jvm.options.d/test.options deleted file mode 100644 index a5030caf5a2e2..0000000000000 --- a/distribution/tools/upgrade-cli/src/test/resources/config/jvm.options.d/test.options +++ /dev/null @@ -1,2 +0,0 @@ --Xms2g --Xmx2g diff --git a/distribution/tools/upgrade-cli/src/test/resources/config/log4j2.properties b/distribution/tools/upgrade-cli/src/test/resources/config/log4j2.properties deleted file mode 100644 index 4b92d3fc62376..0000000000000 --- a/distribution/tools/upgrade-cli/src/test/resources/config/log4j2.properties +++ /dev/null @@ -1,213 +0,0 @@ -status = error - -appender.console.type = Console -appender.console.name = console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -######## Server JSON ############################ -appender.rolling.type = RollingFile -appender.rolling.name = rolling -appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json -appender.rolling.layout.type = ESJsonLayout -appender.rolling.layout.type_name = server - -appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz -appender.rolling.policies.type = Policies -appender.rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.rolling.policies.time.interval = 1 -appender.rolling.policies.time.modulate = true -appender.rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.rolling.policies.size.size = 128MB -appender.rolling.strategy.type = DefaultRolloverStrategy -appender.rolling.strategy.fileIndex = nomax -appender.rolling.strategy.action.type = Delete -appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} -appender.rolling.strategy.action.condition.type = IfFileName -appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* -appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize -appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB -################################################ -######## Server - old style pattern ########### -appender.rolling_old.type = RollingFile -appender.rolling_old.name = rolling_old -appender.rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log -appender.rolling_old.layout.type = PatternLayout -appender.rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -appender.rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz -appender.rolling_old.policies.type = Policies -appender.rolling_old.policies.time.type = TimeBasedTriggeringPolicy -appender.rolling_old.policies.time.interval = 1 -appender.rolling_old.policies.time.modulate = true -appender.rolling_old.policies.size.type = SizeBasedTriggeringPolicy -appender.rolling_old.policies.size.size = 128MB -appender.rolling_old.strategy.type = DefaultRolloverStrategy -appender.rolling_old.strategy.fileIndex = nomax -appender.rolling_old.strategy.action.type = Delete -appender.rolling_old.strategy.action.basepath = ${sys:es.logs.base_path} -appender.rolling_old.strategy.action.condition.type = IfFileName -appender.rolling_old.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* -appender.rolling_old.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize -appender.rolling_old.strategy.action.condition.nested_condition.exceeds = 2GB -################################################ - -rootLogger.level = info -rootLogger.appenderRef.console.ref = console -rootLogger.appenderRef.rolling.ref = rolling -rootLogger.appenderRef.rolling_old.ref = rolling_old - -######## Deprecation JSON ####################### -appender.deprecation_rolling.type = RollingFile -appender.deprecation_rolling.name = deprecation_rolling -appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json -appender.deprecation_rolling.layout.type = ESJsonLayout -appender.deprecation_rolling.layout.type_name = deprecation -appender.deprecation_rolling.layout.esmessagefields=x-opaque-id -appender.deprecation_rolling.filter.rate_limit.type = RateLimitingFilter - -appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.json.gz -appender.deprecation_rolling.policies.type = Policies -appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.deprecation_rolling.policies.size.size = 1GB -appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy -appender.deprecation_rolling.strategy.max = 4 - -appender.header_warning.type = HeaderWarningAppender -appender.header_warning.name = header_warning -################################################# -######## Deprecation - old style pattern ####### -appender.deprecation_rolling_old.type = RollingFile -appender.deprecation_rolling_old.name = deprecation_rolling_old -appender.deprecation_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log -appender.deprecation_rolling_old.layout.type = PatternLayout -appender.deprecation_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -appender.deprecation_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ - _deprecation-%i.log.gz -appender.deprecation_rolling_old.policies.type = Policies -appender.deprecation_rolling_old.policies.size.type = SizeBasedTriggeringPolicy -appender.deprecation_rolling_old.policies.size.size = 1GB -appender.deprecation_rolling_old.strategy.type = DefaultRolloverStrategy -appender.deprecation_rolling_old.strategy.max = 4 -################################################# -logger.deprecation.name = org.elasticsearch.deprecation -logger.deprecation.level = deprecation -logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling -logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling_old -logger.deprecation.appenderRef.header_warning.ref = header_warning -logger.deprecation.additivity = false - -######## Search slowlog JSON #################### -appender.index_search_slowlog_rolling.type = RollingFile -appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling -appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\ - .cluster_name}_index_search_slowlog.json -appender.index_search_slowlog_rolling.layout.type = ESJsonLayout -appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog -appender.index_search_slowlog_rolling.layout.esmessagefields=message,took,took_millis,total_hits,types,stats,search_type,total_shards,source,id - -appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\ - .cluster_name}_index_search_slowlog-%i.json.gz -appender.index_search_slowlog_rolling.policies.type = Policies -appender.index_search_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.index_search_slowlog_rolling.policies.size.size = 1GB -appender.index_search_slowlog_rolling.strategy.type = DefaultRolloverStrategy -appender.index_search_slowlog_rolling.strategy.max = 4 -################################################# -######## Search slowlog - old style pattern #### -appender.index_search_slowlog_rolling_old.type = RollingFile -appender.index_search_slowlog_rolling_old.name = index_search_slowlog_rolling_old -appender.index_search_slowlog_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ - _index_search_slowlog.log -appender.index_search_slowlog_rolling_old.layout.type = PatternLayout -appender.index_search_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -appender.index_search_slowlog_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ - _index_search_slowlog-%i.log.gz -appender.index_search_slowlog_rolling_old.policies.type = Policies -appender.index_search_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy -appender.index_search_slowlog_rolling_old.policies.size.size = 1GB -appender.index_search_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy -appender.index_search_slowlog_rolling_old.strategy.max = 4 -################################################# -logger.index_search_slowlog_rolling.name = index.search.slowlog -logger.index_search_slowlog_rolling.level = trace -logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling -logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling_old.ref = index_search_slowlog_rolling_old -logger.index_search_slowlog_rolling.additivity = false - -######## Indexing slowlog JSON ################## -appender.index_indexing_slowlog_rolling.type = RollingFile -appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling -appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ - _index_indexing_slowlog.json -appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout -appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog -appender.index_indexing_slowlog_rolling.layout.esmessagefields=message,took,took_millis,doc_type,id,routing,source - -appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ - _index_indexing_slowlog-%i.json.gz -appender.index_indexing_slowlog_rolling.policies.type = Policies -appender.index_indexing_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.index_indexing_slowlog_rolling.policies.size.size = 1GB -appender.index_indexing_slowlog_rolling.strategy.type = DefaultRolloverStrategy -appender.index_indexing_slowlog_rolling.strategy.max = 4 -################################################# -######## Indexing slowlog - old style pattern ## -appender.index_indexing_slowlog_rolling_old.type = RollingFile -appender.index_indexing_slowlog_rolling_old.name = index_indexing_slowlog_rolling_old -appender.index_indexing_slowlog_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ - _index_indexing_slowlog.log -appender.index_indexing_slowlog_rolling_old.layout.type = PatternLayout -appender.index_indexing_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -appender.index_indexing_slowlog_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ - _index_indexing_slowlog-%i.log.gz -appender.index_indexing_slowlog_rolling_old.policies.type = Policies -appender.index_indexing_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy -appender.index_indexing_slowlog_rolling_old.policies.size.size = 1GB -appender.index_indexing_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy -appender.index_indexing_slowlog_rolling_old.strategy.max = 4 -################################################# - -logger.index_indexing_slowlog.name = index.indexing.slowlog.index -logger.index_indexing_slowlog.level = trace -logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling -logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling_old.ref = index_indexing_slowlog_rolling_old -logger.index_indexing_slowlog.additivity = false - -######## Task details log JSON #################### -appender.task_detailslog_rolling.type = RollingFile -appender.task_detailslog_rolling.name = task_detailslog_rolling -appender.task_detailslog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_task_detailslog.json -appender.task_detailslog_rolling.layout.type = ESJsonLayout -appender.task_detailslog_rolling.layout.type_name = task_detailslog -appender.task_detailslog_rolling.layout.esmessagefields=taskId,type,action,description,start_time_millis,resource_stats,metadata - -appender.task_detailslog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_task_detailslog-%i.json.gz -appender.task_detailslog_rolling.policies.type = Policies -appender.task_detailslog_rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.task_detailslog_rolling.policies.size.size = 1GB -appender.task_detailslog_rolling.strategy.type = DefaultRolloverStrategy -appender.task_detailslog_rolling.strategy.max = 4 -################################################# -######## Task details log - old style pattern #### -appender.task_detailslog_rolling_old.type = RollingFile -appender.task_detailslog_rolling_old.name = task_detailslog_rolling_old -appender.task_detailslog_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_task_detailslog.log -appender.task_detailslog_rolling_old.layout.type = PatternLayout -appender.task_detailslog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -appender.task_detailslog_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_task_detailslog-%i.log.gz -appender.task_detailslog_rolling_old.policies.type = Policies -appender.task_detailslog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy -appender.task_detailslog_rolling_old.policies.size.size = 1GB -appender.task_detailslog_rolling_old.strategy.type = DefaultRolloverStrategy -appender.task_detailslog_rolling_old.strategy.max = 4 -################################################# -logger.task_detailslog_rolling.name = task.detailslog -logger.task_detailslog_rolling.level = trace -logger.task_detailslog_rolling.appenderRef.task_detailslog_rolling.ref = task_detailslog_rolling -logger.task_detailslog_rolling.appenderRef.task_detailslog_rolling_old.ref = task_detailslog_rolling_old -logger.task_detailslog_rolling.additivity = false diff --git a/settings.gradle b/settings.gradle index 1fb5795b1a8ba..a1cd3c76e5c22 100644 --- a/settings.gradle +++ b/settings.gradle @@ -77,7 +77,6 @@ List projects = [ 'distribution:tools:launchers', 'distribution:tools:plugin-cli', 'distribution:tools:keystore-cli', - 'distribution:tools:upgrade-cli', 'server', 'server:cli', 'test:framework', From 94143ad23546af7aff37b69556aa1c8c90154e65 Mon Sep 17 00:00:00 2001 From: Shahbaz Aamir Date: Tue, 15 Jul 2025 23:28:43 +0530 Subject: [PATCH 48/51] Improve test instructions in developer guide (#18736) Signed-off-by: Shahbaz Aamir --- DEVELOPER_GUIDE.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 016a4f9900402..2d7125b241af7 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -189,7 +189,9 @@ It's typically easier to wait until the console stops scrolling, and then run `c ```bash curl localhost:9200 - +``` +The expected reponse should be +``` { "name" : "runTask-0", "cluster_name" : "runTask", From 6f9e1f8c39e9ee9117667e538108c4351a7bc8c6 Mon Sep 17 00:00:00 2001 From: Michael Froh Date: Tue, 15 Jul 2025 14:07:39 -0700 Subject: [PATCH 49/51] [Experimental] Start without joining a cluster if a "clusterless" ClusterPlugin is loaded (#18479) These are the core changes to allow ClusterPlugins to manage cluster state on behalf of an OpenSearch node. If a "clusterless" ClusterPlugin is loaded, then the node will start up with dummy versions of Discovery and ClusterManagerService, and will load the minimum cluster state required for startup (essentially, the node needs to see itself). From there, the ClusterPlugin can interact with ClusterApplierService to apply some cluster state to the node. (The details of constructing/receiving that cluster state is up to the plugin.) --------- Signed-off-by: Michael Froh --- CHANGELOG.md | 1 + .../org/opensearch/cluster/ClusterModule.java | 11 ++- .../action/shard/LocalShardStateAction.java | 88 +++++++++++++++++++ .../action/shard/ShardStateAction.java | 4 +- .../service/ClusterApplierService.java | 8 ++ .../cluster/service/LocalClusterService.java | 87 ++++++++++++++++++ .../opensearch/discovery/LocalDiscovery.java | 82 +++++++++++++++++ .../main/java/org/opensearch/node/Node.java | 73 ++++++++------- .../org/opensearch/plugins/ClusterPlugin.java | 9 ++ .../cluster/ClusterModuleTests.java | 35 +++++--- 10 files changed, 353 insertions(+), 45 deletions(-) create mode 100644 server/src/main/java/org/opensearch/cluster/action/shard/LocalShardStateAction.java create mode 100644 server/src/main/java/org/opensearch/cluster/service/LocalClusterService.java create mode 100644 server/src/main/java/org/opensearch/discovery/LocalDiscovery.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d04c01edc47d9..1a7fd77591249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Pass index settings to system ingest processor factories. ([#18708](https://github.com/opensearch-project/OpenSearch/pull/18708)) - Include named queries from rescore contexts in matched_queries array ([#18697](https://github.com/opensearch-project/OpenSearch/pull/18697)) - Add the configurable limit on rule cardinality ([#18663](https://github.com/opensearch-project/OpenSearch/pull/18663)) +- [Experimental] Start in "clusterless" mode if a clusterless ClusterPlugin is loaded ([#18479](https://github.com/opensearch-project/OpenSearch/pull/18479)) ### Changed - Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570)) diff --git a/server/src/main/java/org/opensearch/cluster/ClusterModule.java b/server/src/main/java/org/opensearch/cluster/ClusterModule.java index 7dbc0d0ee88e3..517d3abd2a472 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterModule.java @@ -144,6 +144,7 @@ public class ClusterModule extends AbstractModule { final Collection deciderList; final ShardsAllocator shardsAllocator; private final ClusterManagerMetrics clusterManagerMetrics; + private final Class shardStateActionClass; public ClusterModule( Settings settings, @@ -152,7 +153,8 @@ public ClusterModule( ClusterInfoService clusterInfoService, SnapshotsInfoService snapshotsInfoService, ThreadContext threadContext, - ClusterManagerMetrics clusterManagerMetrics + ClusterManagerMetrics clusterManagerMetrics, + Class shardStateActionClass ) { this.clusterPlugins = clusterPlugins; this.deciderList = createAllocationDeciders(settings, clusterService.getClusterSettings(), clusterPlugins); @@ -169,6 +171,7 @@ public ClusterModule( clusterManagerMetrics ); this.clusterManagerMetrics = clusterManagerMetrics; + this.shardStateActionClass = shardStateActionClass; } public static List getNamedWriteables() { @@ -474,7 +477,11 @@ protected void configure() { bind(MetadataIndexTemplateService.class).asEagerSingleton(); bind(IndexNameExpressionResolver.class).toInstance(indexNameExpressionResolver); bind(DelayedAllocationService.class).asEagerSingleton(); - bind(ShardStateAction.class).asEagerSingleton(); + if (shardStateActionClass == ShardStateAction.class) { + bind(ShardStateAction.class).asEagerSingleton(); + } else { + bind(ShardStateAction.class).to(shardStateActionClass).asEagerSingleton(); + } bind(NodeMappingRefreshAction.class).asEagerSingleton(); bind(MappingUpdatedAction.class).asEagerSingleton(); bind(TaskResultsService.class).asEagerSingleton(); diff --git a/server/src/main/java/org/opensearch/cluster/action/shard/LocalShardStateAction.java b/server/src/main/java/org/opensearch/cluster/action/shard/LocalShardStateAction.java new file mode 100644 index 0000000000000..335cede540b57 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/action/shard/LocalShardStateAction.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.action.shard; + +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.RerouteService; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.allocation.AllocationService; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.util.function.Function; + +/** + * A local implementation of {@link ShardStateAction} that applies shard state changes directly to the + * local cluster state. This is used in clusterless mode, where there is no cluster manager. + */ +public class LocalShardStateAction extends ShardStateAction { + @Inject + public LocalShardStateAction( + ClusterService clusterService, + TransportService transportService, + AllocationService allocationService, + RerouteService rerouteService, + ThreadPool threadPool + ) { + super(clusterService, transportService, allocationService, rerouteService, threadPool); + } + + @Override + public void shardStarted( + ShardRouting shardRouting, + long primaryTerm, + String message, + ActionListener listener, + ClusterState currentState + ) { + Function clusterStateUpdater = clusterState -> { + // We're running in clusterless mode. Apply the state change directly to the local cluster state. + RoutingTable routingTable = clusterState.getRoutingTable(); + IndexRoutingTable indexRoutingTable = routingTable.index(shardRouting.index()); + + ClusterState.Builder clusterStateBuilder = ClusterState.builder(clusterState); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(routingTable); + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(shardRouting.index()); + for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { + if (indexShardRoutingTable.shardId().equals(shardRouting.shardId())) { + IndexShardRoutingTable.Builder indexShardRoutingTableBuilder = new IndexShardRoutingTable.Builder( + indexShardRoutingTable + ); + indexShardRoutingTableBuilder.removeShard(shardRouting); + indexShardRoutingTableBuilder.addShard(shardRouting.moveToStarted()); + indexRoutingTableBuilder.addIndexShard(indexShardRoutingTableBuilder.build()); + } else { + indexRoutingTableBuilder.addIndexShard(indexShardRoutingTable); + } + } + routingTableBuilder.add(indexRoutingTableBuilder); + clusterStateBuilder.routingTable(routingTableBuilder.build()); + return clusterStateBuilder.build(); + }; + clusterService.getClusterApplierService() + .updateClusterState("shard-started " + shardRouting.shardId(), clusterStateUpdater, (s, e) -> {}); + } + + @Override + public void localShardFailed( + ShardRouting shardRouting, + String message, + Exception failure, + ActionListener listener, + ClusterState currentState + ) { + // Do not send a failure to the cluster manager, as we are running in clusterless mode. + } +} diff --git a/server/src/main/java/org/opensearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/opensearch/cluster/action/shard/ShardStateAction.java index cb5749a91d448..6a204925ccd04 100644 --- a/server/src/main/java/org/opensearch/cluster/action/shard/ShardStateAction.java +++ b/server/src/main/java/org/opensearch/cluster/action/shard/ShardStateAction.java @@ -88,7 +88,7 @@ import java.util.function.Supplier; /** - * Transport action for retrieving the shard state + * Registers transport actions that react to shard state changes, such as shard started or shard failed. * * @opensearch.internal */ @@ -128,7 +128,7 @@ private static Priority parseReroutePriority(String priorityString) { } private final TransportService transportService; - private final ClusterService clusterService; + final ClusterService clusterService; private final ThreadPool threadPool; private volatile Priority followUpRerouteTaskPriority; diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java index 7ab1a082a4620..fc2a121c90e54 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java @@ -382,6 +382,14 @@ public void onNewClusterState( submitStateUpdateTask(source, ClusterStateTaskConfig.build(Priority.HIGH), applyFunction, listener); } + public void updateClusterState( + final String source, + final Function updateFunction, + final ClusterApplyListener listener + ) { + submitStateUpdateTask(source, ClusterStateTaskConfig.build(Priority.HIGH), updateFunction, listener); + } + private void submitStateUpdateTask( final String source, final ClusterStateTaskConfig config, diff --git a/server/src/main/java/org/opensearch/cluster/service/LocalClusterService.java b/server/src/main/java/org/opensearch/cluster/service/LocalClusterService.java new file mode 100644 index 0000000000000..4caf37cacfc42 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/service/LocalClusterService.java @@ -0,0 +1,87 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.service; + +import org.opensearch.cluster.ClusterManagerMetrics; +import org.opensearch.cluster.ClusterStateTaskConfig; +import org.opensearch.cluster.ClusterStateTaskExecutor; +import org.opensearch.cluster.ClusterStateTaskListener; +import org.opensearch.cluster.coordination.ClusterStatePublisher; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.node.Node; +import org.opensearch.threadpool.ThreadPool; + +import java.util.Map; + +/** + * A local implementation of {@link ClusterService} that assumes we have no cluster manager. + * This is used in clusterless mode. + */ +public class LocalClusterService extends ClusterService { + private static class DummyClusterManagerService extends ClusterManagerService { + private static final ClusterManagerThrottlingStats EMPTY_THROTTLING_STATS = new ClusterManagerThrottlingStats(); + + public DummyClusterManagerService(Settings settings, ClusterSettings clusterSettings) { + super(settings, clusterSettings, null, null); + } + + @Override + public synchronized void setClusterStatePublisher(ClusterStatePublisher publisher) {} + + @Override + public ClusterManagerThrottlingStats getThrottlingStats() { + return EMPTY_THROTTLING_STATS; + } + } + + public LocalClusterService( + Settings settings, + ClusterSettings clusterSettings, + ThreadPool threadPool, + ClusterManagerMetrics clusterManagerMetrics + ) { + super( + settings, + clusterSettings, + new DummyClusterManagerService(settings, clusterSettings), + new ClusterApplierService(Node.NODE_NAME_SETTING.get(settings), settings, clusterSettings, threadPool, clusterManagerMetrics) + ); + } + + @Override + protected synchronized void doStart() { + getClusterApplierService().start(); + } + + @Override + protected synchronized void doStop() { + getClusterApplierService().stop(); + } + + @Override + protected synchronized void doClose() { + getClusterApplierService().close(); + } + + @Override + public ClusterManagerTaskThrottler.ThrottlingKey registerClusterManagerTask(ClusterManagerTask task, boolean throttlingEnabled) { + return null; + } + + @Override + public void submitStateUpdateTasks( + final String source, + final Map tasks, + final ClusterStateTaskConfig config, + final ClusterStateTaskExecutor executor + ) { + throw new UnsupportedOperationException("Cannot submit cluster state update tasks when cluster manager service is not available"); + } +} diff --git a/server/src/main/java/org/opensearch/discovery/LocalDiscovery.java b/server/src/main/java/org/opensearch/discovery/LocalDiscovery.java new file mode 100644 index 0000000000000..d0090f5b4e7f4 --- /dev/null +++ b/server/src/main/java/org/opensearch/discovery/LocalDiscovery.java @@ -0,0 +1,82 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.discovery; + +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.NodeConnectionsService; +import org.opensearch.cluster.coordination.PendingClusterStateStats; +import org.opensearch.cluster.coordination.PublishClusterStateStats; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterApplier; +import org.opensearch.cluster.service.ClusterStateStats; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; +import org.opensearch.core.action.ActionListener; +import org.opensearch.transport.TransportService; + +import java.io.IOException; + +/** + * Clusterless implementation of Discovery. This is only able to "discover" the local node. + */ +public class LocalDiscovery extends AbstractLifecycleComponent implements Discovery { + private static final DiscoveryStats EMPTY_STATS = new DiscoveryStats( + new PendingClusterStateStats(0, 0, 0), + new PublishClusterStateStats(0, 0, 0), + new ClusterStateStats() + ); + private final TransportService transportService; + private final ClusterApplier clusterApplier; + + public LocalDiscovery(TransportService transportService, ClusterApplier clusterApplier) { + this.transportService = transportService; + this.clusterApplier = clusterApplier; + } + + @Override + public void publish(ClusterChangedEvent clusterChangedEvent, ActionListener publishListener, AckListener ackListener) { + // In clusterless mode, we should never be asked to publish a cluster state. + throw new UnsupportedOperationException("Should not be called in clusterless mode"); + } + + @Override + protected void doStart() { + DiscoveryNode localNode = transportService.getLocalNode(); + ClusterState bootstrapClusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .nodes(DiscoveryNodes.builder().localNodeId(localNode.getId()).add(localNode).build()) + .build(); + clusterApplier.setInitialState(bootstrapClusterState); + } + + @Override + protected void doStop() { + + } + + @Override + protected void doClose() throws IOException { + + } + + @Override + public DiscoveryStats stats() { + return EMPTY_STATS; + } + + @Override + public void startInitialJoin() { + + } + + @Override + public void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService) { + + } +} diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index a5e92293c0be1..5b3559b2fde49 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -68,6 +68,8 @@ import org.opensearch.cluster.InternalClusterInfoService; import org.opensearch.cluster.NodeConnectionsService; import org.opensearch.cluster.action.index.MappingUpdatedAction; +import org.opensearch.cluster.action.shard.LocalShardStateAction; +import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.applicationtemplates.SystemTemplatesPlugin; import org.opensearch.cluster.applicationtemplates.SystemTemplatesService; import org.opensearch.cluster.coordination.PersistedStateRegistry; @@ -86,6 +88,7 @@ import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.cluster.routing.allocation.DiskThresholdMonitor; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.cluster.service.LocalClusterService; import org.opensearch.common.SetOnce; import org.opensearch.common.StopWatch; import org.opensearch.common.cache.module.CacheModule; @@ -130,6 +133,7 @@ import org.opensearch.crypto.CryptoHandlerRegistry; import org.opensearch.discovery.Discovery; import org.opensearch.discovery.DiscoveryModule; +import org.opensearch.discovery.LocalDiscovery; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.NodeMetadata; @@ -702,12 +706,13 @@ protected Node(final Environment initialEnvironment, Collection clas final ClusterManagerMetrics clusterManagerMetrics = new ClusterManagerMetrics(metricsRegistry); List clusterPlugins = pluginsService.filterPlugins(ClusterPlugin.class); - final ClusterService clusterService = new ClusterService( - settings, - settingsModule.getClusterSettings(), - threadPool, - clusterManagerMetrics - ); + final boolean clusterless = clusterPlugins.stream().anyMatch(ClusterPlugin::isClusterless); + final ClusterService clusterService; + if (clusterless) { + clusterService = new LocalClusterService(settings, settingsModule.getClusterSettings(), threadPool, clusterManagerMetrics); + } else { + clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool, clusterManagerMetrics); + } clusterService.addStateApplier(scriptService); resourcesToClose.add(clusterService); final Set> consistentSettings = settingsModule.getConsistentSettings(); @@ -766,7 +771,8 @@ protected Node(final Environment initialEnvironment, Collection clas clusterInfoService, snapshotsInfoService, threadPool.getThreadContext(), - clusterManagerMetrics + clusterManagerMetrics, + clusterless ? LocalShardStateAction.class : ShardStateAction.class ); modules.add(clusterModule); final List mapperPlugins = pluginsService.filterPlugins(MapperPlugin.class); @@ -1347,27 +1353,31 @@ protected Node(final Environment initialEnvironment, Collection clas ); clusterInfoService.addListener(diskThresholdMonitor::onNewInfo); - final DiscoveryModule discoveryModule = new DiscoveryModule( - settings, - threadPool, - transportService, - namedWriteableRegistry, - networkService, - clusterService.getClusterManagerService(), - clusterService.getClusterApplierService(), - clusterService.getClusterSettings(), - pluginsService.filterPlugins(DiscoveryPlugin.class), - clusterModule.getAllocationService(), - environment.configDir(), - gatewayMetaState, - rerouteService, - fsHealthService, - persistedStateRegistry, - remoteStoreNodeService, - clusterManagerMetrics, - remoteClusterStateService - ); - + final Discovery discovery; + if (clusterless) { + discovery = new LocalDiscovery(transportService, clusterService.getClusterApplierService()); + } else { + discovery = new DiscoveryModule( + settings, + threadPool, + transportService, + namedWriteableRegistry, + networkService, + clusterService.getClusterManagerService(), + clusterService.getClusterApplierService(), + clusterService.getClusterSettings(), + pluginsService.filterPlugins(DiscoveryPlugin.class), + clusterModule.getAllocationService(), + environment.configDir(), + gatewayMetaState, + rerouteService, + fsHealthService, + persistedStateRegistry, + remoteStoreNodeService, + clusterManagerMetrics, + remoteClusterStateService + ).getDiscovery(); + } final SearchPipelineService searchPipelineService = new SearchPipelineService( clusterService, threadPool, @@ -1388,11 +1398,12 @@ protected Node(final Environment initialEnvironment, Collection clas transportService.getTaskManager(), taskCancellationMonitoringSettings ); + this.nodeService = new NodeService( settings, threadPool, monitorService, - discoveryModule.getDiscovery(), + discovery, transportService, indicesService, pluginsService, @@ -1535,7 +1546,7 @@ protected Node(final Environment initialEnvironment, Collection clas b.bind(ClusterInfoService.class).toInstance(clusterInfoService); b.bind(SnapshotsInfoService.class).toInstance(snapshotsInfoService); b.bind(GatewayMetaState.class).toInstance(gatewayMetaState); - b.bind(Discovery.class).toInstance(discoveryModule.getDiscovery()); + b.bind(Discovery.class).toInstance(discovery); b.bind(RemoteStoreSettings.class).toInstance(remoteStoreSettings); { b.bind(PeerRecoverySourceService.class) @@ -1716,7 +1727,7 @@ public Node start() throws NodeValidationException { injector.getInstance(GatewayService.class).start(); Discovery discovery = injector.getInstance(Discovery.class); discovery.setNodeConnectionsService(nodeConnectionsService); - clusterService.getClusterManagerService().setClusterStatePublisher(discovery::publish); + clusterService.getClusterManagerService().setClusterStatePublisher(discovery); // Start the transport service now so the publish address will be added to the local disco node in ClusterService TransportService transportService = injector.getInstance(TransportService.class); diff --git a/server/src/main/java/org/opensearch/plugins/ClusterPlugin.java b/server/src/main/java/org/opensearch/plugins/ClusterPlugin.java index 9b2ddd0bed41a..cfd26814697e0 100644 --- a/server/src/main/java/org/opensearch/plugins/ClusterPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/ClusterPlugin.java @@ -111,4 +111,13 @@ default void onNodeStarted(DiscoveryNode localNode) { onNodeStarted(); } + /** + * @return true if this plugin will handle cluster state management on behalf of the node, so the node does not + * need to discover a cluster manager and be part of a cluster. + * + * Note that if any ClusterPlugin returns true from this method, the node will start in clusterless mode. + */ + default boolean isClusterless() { + return false; + } } diff --git a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java index 82551bc26e224..3b20af8446704 100644 --- a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java +++ b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java @@ -32,6 +32,7 @@ package org.opensearch.cluster; +import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; @@ -91,7 +92,7 @@ import java.util.function.Supplier; public class ClusterModuleTests extends ModuleTestCase { - private ClusterInfoService clusterInfoService = EmptyClusterInfoService.INSTANCE; + private final ClusterInfoService clusterInfoService = EmptyClusterInfoService.INSTANCE; private ClusterService clusterService; private ThreadContext threadContext; @@ -158,7 +159,8 @@ public Collection getIndexNameCu clusterInfoService, null, threadContext, - null + null, + ShardStateAction.class ); assertTrue(module.getIndexNameExpressionResolver().getExpressionResolvers().contains(customResolver1)); assertTrue(module.getIndexNameExpressionResolver().getExpressionResolvers().contains(customResolver2)); @@ -175,7 +177,16 @@ public Collection getIndexNameCu }); IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, - () -> new ClusterModule(Settings.EMPTY, clusterService, clusterPlugins, clusterInfoService, null, threadContext, null) + () -> new ClusterModule( + Settings.EMPTY, + clusterService, + clusterPlugins, + clusterInfoService, + null, + threadContext, + null, + ShardStateAction.class + ) ); assertEquals( "Cannot specify expression resolver [org.opensearch.cluster.ClusterModuleTests$FakeExpressionResolver] twice", @@ -229,7 +240,7 @@ public void testRegisterAllocationDeciderDuplicate() { public Collection createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) { return Collections.singletonList(new EnableAllocationDecider(settings, clusterSettings)); } - }), clusterInfoService, null, threadContext, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)) + }), clusterInfoService, null, threadContext, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE), ShardStateAction.class) ); assertEquals(e.getMessage(), "Cannot specify allocation decider [" + EnableAllocationDecider.class.getName() + "] twice"); } @@ -240,7 +251,7 @@ public void testRegisterAllocationDecider() { public Collection createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) { return Collections.singletonList(new FakeAllocationDecider()); } - }), clusterInfoService, null, threadContext, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)); + }), clusterInfoService, null, threadContext, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE), ShardStateAction.class); assertTrue(module.deciderList.stream().anyMatch(d -> d.getClass().equals(FakeAllocationDecider.class))); } @@ -250,7 +261,7 @@ private ClusterModule newClusterModuleWithShardsAllocator(Settings settings, Str public Map> getShardsAllocators(Settings settings, ClusterSettings clusterSettings) { return Collections.singletonMap(name, supplier); } - }), clusterInfoService, null, threadContext, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)); + }), clusterInfoService, null, threadContext, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE), ShardStateAction.class); } public void testRegisterShardsAllocator() { @@ -278,7 +289,8 @@ public void testUnknownShardsAllocator() { clusterInfoService, null, threadContext, - new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE) + new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE), + ShardStateAction.class ) ); assertEquals("Unknown ShardsAllocator [dne]", e.getMessage()); @@ -366,7 +378,8 @@ public void testRejectsReservedExistingShardsAllocatorName() { clusterInfoService, null, threadContext, - new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE) + new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE), + ShardStateAction.class ); expectThrows( IllegalArgumentException.class, @@ -382,7 +395,8 @@ public void testRejectsDuplicateExistingShardsAllocatorName() { clusterInfoService, null, threadContext, - new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE) + new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE), + ShardStateAction.class ); expectThrows( IllegalArgumentException.class, @@ -415,7 +429,8 @@ public void testRerouteServiceSetForBalancedShardsAllocator() { clusterInfoService, null, threadContext, - new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE) + new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE), + ShardStateAction.class ); clusterModule.setRerouteServiceForAllocator((reason, priority, listener) -> listener.onResponse(clusterService.state())); } From cb22cb02c9f9a4acd40f4c460ce9247a4c5b127b Mon Sep 17 00:00:00 2001 From: rayshrey <121871912+rayshrey@users.noreply.github.com> Date: Wed, 16 Jul 2025 03:06:59 +0530 Subject: [PATCH 50/51] Check CircuitBreaker before put/compute in FileCache to avoid entry removal (#18661) Signed-off-by: Shreyansh Ray --- .../store/remote/filecache/FileCache.java | 8 +++----- .../store/remote/filecache/FileCacheTests.java | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java index 718f79a74f9d7..b8b1e3c650b1b 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java @@ -69,8 +69,8 @@ public long capacity() { @Override public CachedIndexInput put(Path filePath, CachedIndexInput indexInput) { + checkParentBreaker(); CachedIndexInput cachedIndexInput = theCache.put(filePath, indexInput); - checkParentBreaker(filePath); return cachedIndexInput; } @@ -79,8 +79,8 @@ public CachedIndexInput compute( Path key, BiFunction remappingFunction ) { + checkParentBreaker(); CachedIndexInput cachedIndexInput = theCache.compute(key, remappingFunction); - checkParentBreaker(key); return cachedIndexInput; } @@ -201,13 +201,11 @@ public void closeIndexInputReferences() { /** * Ensures that the PARENT breaker is not tripped when an entry is added to the cache - * @param filePath the path key for which entry is added */ - private void checkParentBreaker(Path filePath) { + private void checkParentBreaker() { try { circuitBreaker.addEstimateBytesAndMaybeBreak(0, "filecache_entry"); } catch (CircuitBreakingException ex) { - theCache.remove(filePath); throw new CircuitBreakingException( "Unable to create file cache entries", ex.getBytesWanted(), diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java index 3ab193c5325c6..3e4be7ef5c294 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java @@ -50,6 +50,10 @@ private FileCache createFileCache(long capacity) { return FileCacheFactory.createConcurrentLRUFileCache(capacity, CONCURRENCY_LEVEL, new NoopCircuitBreaker(CircuitBreaker.REQUEST)); } + private FileCache createFileCache(long capacity, CircuitBreaker circuitBreaker) { + return FileCacheFactory.createConcurrentLRUFileCache(capacity, CONCURRENCY_LEVEL, circuitBreaker); + } + private FileCache createCircuitBreakingFileCache(long capacity) { TestCircuitBreaker testCircuitBreaker = new TestCircuitBreaker(); testCircuitBreaker.startBreaking(); @@ -204,6 +208,20 @@ public void testComputeThrowCircuitBreakingException() { assertNull(fileCache.get(path)); } + public void testEntryNotRemovedCircuitBreaker() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + FileCache fileCache = createFileCache(MEGA_BYTES, circuitBreaker); + Path path = createPath("0"); + fileCache.put(path, new StubCachedIndexInput(8 * MEGA_BYTES)); + // put should succeed since circuit breaker hasn't tripped yet + assertEquals(fileCache.get(path).length(), 8 * MEGA_BYTES); + circuitBreaker.startBreaking(); + // compute should throw CircuitBreakingException but shouldn't remove entry already present + assertThrows(CircuitBreakingException.class, () -> fileCache.compute(path, (p, i) -> new StubCachedIndexInput(2 * MEGA_BYTES))); + assertNotNull(fileCache.get(path)); + assertEquals(fileCache.get(path).length(), 8 * MEGA_BYTES); + } + public void testRemove() { FileCache fileCache = createFileCache(MEGA_BYTES); for (int i = 0; i < 4; i++) { From c7c648fef06be76bdea813a7dde1d99338d8c300 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 15 Jul 2025 22:13:25 -0400 Subject: [PATCH 51/51] Fix UNIX domain socket permission checks on Windows (#18764) Signed-off-by: Andriy Redko --- .../opensearch/javaagent/FileInterceptor.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/FileInterceptor.java b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/FileInterceptor.java index f52f1bc91f559..455be2a83f840 100644 --- a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/FileInterceptor.java +++ b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/FileInterceptor.java @@ -74,8 +74,23 @@ public static void intercept(@Advice.AllArguments Object[] args, @Advice.Origin final boolean isDelete = isMutating == false ? name.startsWith("delete") : false; // This is Windows implementation of UNIX Domain Sockets (close) - if (isDelete == true - && walker.getCallerClass().getName().equalsIgnoreCase("sun.nio.ch.PipeImpl$Initializer$LoopbackConnector") == true) { + boolean isUnixSocketCaller = false; + if (isDelete == true) { + final Collection> chain = walker.walk(StackCallerClassChainExtractor.INSTANCE); + + if (walker.getCallerClass().getName().equalsIgnoreCase("sun.nio.ch.PipeImpl$Initializer$LoopbackConnector") == true) { + isUnixSocketCaller = true; + } else { + for (final Class caller : chain) { + if (caller.getName().equalsIgnoreCase("sun.nio.ch.PipeImpl$Initializer$LoopbackConnector")) { + isUnixSocketCaller = true; + break; + } + } + } + } + + if (isDelete == true && isUnixSocketCaller == true) { final NetPermission permission = new NetPermission("accessUnixDomainSocket"); for (ProtectionDomain domain : callers) { if (!policy.implies(domain, permission)) {