Merge branch 'main' into remove-socket-access

Signed-off-by: Craig Perkins <[email protected]>

Author: cwperks

.github/workflows/assemble.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       matrix:
         java: [ 21, 25 ]
-        os: [ubuntu-latest, windows-latest, macos-13, ubuntu-24.04-arm]
+        os: [ubuntu-latest, windows-latest, macos-15, ubuntu-24.04-arm]
     steps:
       - uses: actions/checkout@v5
       - name: Set up JDK ${{ matrix.java }}

.github/workflows/gradle-check.yml

@@ -24,7 +24,7 @@ jobs:
     - uses: actions/checkout@v5
     - name: Get changed files
       id: changed-files-specific
-      uses: tj-actions/changed-files@v46.0.5
+      uses: tj-actions/changed-files@v47.0.0
       with:
         files_ignore: |
           release-notes/*.md

.github/workflows/precommit.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       matrix:
         java: [ 21, 25 ]
-        os: [ubuntu-latest, windows-latest, macos-latest, macos-13, ubuntu-24.04-arm]
+        os: [ubuntu-latest, windows-latest, macos-15, macos-15-intel, ubuntu-24.04-arm]
         include:
           - java: 21
             os: 'windows-2025'

CHANGELOG.md

@@ -5,7 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 3.x]
 ### Added
-
 - Add support for a ForkJoinPool type ([#19008](https://github.com/opensearch-project/OpenSearch/pull/19008))
 - Add seperate shard limit validation for local and remote indices ([#19532](https://github.com/opensearch-project/OpenSearch/pull/19532))
 - Use Lucene `pack` method for `half_float` and `usigned_long` when using `ApproximatePointRangeQuery`.
@@ -15,11 +14,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add pluggable gRPC interceptors with explicit ordering([#19005](https://github.com/opensearch-project/OpenSearch/pull/19005))
 - Add BindableServices extension point to transport-grpc-spi ([#19304](https://github.com/opensearch-project/OpenSearch/pull/19304))
 - Add metrics for the merged segment warmer feature ([#18929](https://github.com/opensearch-project/OpenSearch/pull/18929))
+- Handle deleted documents for filter rewrite subaggregation optimization ([#19643](https://github.com/opensearch-project/OpenSearch/pull/19643))
 - Add pointer based lag metric in pull-based ingestion ([#19635](https://github.com/opensearch-project/OpenSearch/pull/19635))
 - Introduced internal API for retrieving metadata about requested indices from transport actions  ([#18523](https://github.com/opensearch-project/OpenSearch/pull/18523))
 - Add cluster defaults for merge autoThrottle, maxMergeThreads, and maxMergeCount; Add segment size filter to the merged segment warmer ([#19629](https://github.com/opensearch-project/OpenSearch/pull/19629))
 - Add build-tooling to run in FIPS environment ([#18921](https://github.com/opensearch-project/OpenSearch/pull/18921))
 - Add SMILE/CBOR/YAML document format support to Bulk GRPC endpoint ([#19744](https://github.com/opensearch-project/OpenSearch/pull/19744))
+- Implement GRPC Search params `Highlight`and `Sort` ([#19868](https://github.com/opensearch-project/OpenSearch/pull/19868))
+- Implement GRPC ConstantScoreQuery, FuzzyQuery, MatchBoolPrefixQuery, MatchPhrasePrefix, PrefixQuery, MatchQuery ([#19854](https://github.com/opensearch-project/OpenSearch/pull/19854))
+- Add async periodic flush task support for pull-based ingestion ([#19878](https://github.com/opensearch-project/OpenSearch/pull/19878))
+- Add support for context aware segments ([#19098](https://github.com/opensearch-project/OpenSearch/pull/19098))
+- Implement GRPC FunctionScoreQuery ([#19888](https://github.com/opensearch-project/OpenSearch/pull/19888))
 
 ### Changed
 - Faster `terms` query creation for `keyword` field with index and docValues enabled ([#19350](https://github.com/opensearch-project/OpenSearch/pull/19350))
@@ -33,6 +38,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Refactor the IndexingStats.Stats class to use the Builder pattern instead of constructors ([#19306](https://github.com/opensearch-project/OpenSearch/pull/19306))
 - Remove FeatureFlag.MERGED_SEGMENT_WARMER_EXPERIMENTAL_FLAG. ([#19715](https://github.com/opensearch-project/OpenSearch/pull/19715))
 - Replace java.security.AccessController with org.opensearch.secure_sm.AccessController in sub projects with SocketAccess class ([#19803](https://github.com/opensearch-project/OpenSearch/pull/19803))
+- Replace java.security.AccessController with org.opensearch.secure_sm.AccessController in discovery plugins ([#19802](https://github.com/opensearch-project/OpenSearch/pull/19802))
+- Change the default value of doc_values in WildcardFieldMapper to true. ([#19796](https://github.com/opensearch-project/OpenSearch/pull/19796))
+- Make Engine#loadHistoryUUID() protected and Origin#isFromTranslog() public ([#19753](https://github.com/opensearch-project/OpenSearch/pull/19752))
+- Bump opensearch-protobufs dependency to 0.23.0 and update transport-grpc module compatibility ([#19831](https://github.com/opensearch-project/OpenSearch/pull/19831))
+- Refactor the RefreshStats class to use the Builder pattern instead of constructors ([#19835](https://github.com/opensearch-project/OpenSearch/pull/19835))
+- Refactor the DocStats and StoreStats class to use the Builder pattern instead of constructors ([#19863](https://github.com/opensearch-project/OpenSearch/pull/19863))
+- Pass registry of headers from ActionPlugin.getRestHeaders to ActionPlugin.getRestHandlerWrapper ([#19875](https://github.com/opensearch-project/OpenSearch/pull/19875))
 
 ### Fixed
 - Fix Allocation and Rebalance Constraints of WeightFunction are incorrectly reset ([#19012](https://github.com/opensearch-project/OpenSearch/pull/19012))
@@ -46,10 +58,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix file-based ingestion consumer to handle start point beyond max line number([#19757])(https://github.com/opensearch-project/OpenSearch/pull/19757))
 - Fix IndexOutOfBoundsException when running include/exclude on non-existent prefix in terms aggregations ([#19637](https://github.com/opensearch-project/OpenSearch/pull/19637))
 - Fixed assertion unsafe use of ClusterService.state() in ResourceUsageCollectorService ([#19775])(https://github.com/opensearch-project/OpenSearch/pull/19775))
+- Fix Unified highlighter for nested fields when using matchPhrasePrefixQuery ([#19442](https://github.com/opensearch-project/OpenSearch/pull/19442))
 - Add S3Repository.LEGACY_MD5_CHECKSUM_CALCULATION to list of repository-s3 settings ([#19788](https://github.com/opensearch-project/OpenSearch/pull/19788))
+- Fix NPE of ScriptScoreQuery ([#19650](https://github.com/opensearch-project/OpenSearch/pull/19650))
 
 ### Dependencies
-- Update to Gradle 9.1 ([#19575](https://github.com/opensearch-project/OpenSearch/pull/19575))
+- Update to Gradle 9.2 ([#19575](https://github.com/opensearch-project/OpenSearch/pull/19575)) ([#19856](https://github.com/opensearch-project/OpenSearch/pull/19856))
+- Update bundled JDK to JDK-25 ([#19314](https://github.com/opensearch-project/OpenSearch/issues/19314))
 - Bump `peter-evans/create-or-update-comment` from 4 to 5 ([#19536](https://github.com/opensearch-project/OpenSearch/pull/19536))
 - Bump `com.azure:azure-core-http-netty` from 1.15.12 to 1.16.1 ([#19533](https://github.com/opensearch-project/OpenSearch/pull/19533))
 - Bump `org.apache.zookeeper:zookeeper` from 3.9.3 to 3.9.4 ([#19535](https://github.com/opensearch-project/OpenSearch/pull/19535))
@@ -64,10 +79,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump ch.qos.logback modules from 1.5.18 to 1.5.20 in HDFS test fixture ([#19763](https://github.com/opensearch-project/OpenSearch/pull/19763))
 - Bump `github/codeql-action` from 3 to 4 ([#19785](https://github.com/opensearch-project/OpenSearch/pull/19785))
 - Bump `gradle/actions` from 4 to 5 ([#19781](https://github.com/opensearch-project/OpenSearch/pull/19781))
+- Bump org.bouncycastle:bc-fips from 2.1.1 to 2.1.2 ([#19818](https://github.com/opensearch-project/OpenSearch/pull/19818))
+- Bump `com.google.http-client:google-http-client-gson` from 1.47.1 to 2.0.0 ([#19253](https://github.com/opensearch-project/OpenSearch/pull/19253))
+- Bump `com.sun.xml.bind:jaxb-impl` from 2.2.3-1 to 4.0.6 ([#19472](https://github.com/opensearch-project/OpenSearch/pull/19472))
+- Bump `tj-actions/changed-files` from 46.0.5 to 47.0.0 ([#19873](https://github.com/opensearch-project/OpenSearch/pull/19873))
+- Bump `com.google.api.grpc:proto-google-iam-v1` from 1.55.0 to 1.57.0 ([#19872](https://github.com/opensearch-project/OpenSearch/pull/19872))
+- Bump `org.apache.commons:commons-text` from 1.13.1 to 1.14.0 ([#19871](https://github.com/opensearch-project/OpenSearch/pull/19871))
+- Exclude group com.microsoft.sqlserver from hadoop-minicluster ([#19889](https://github.com/opensearch-project/OpenSearch/pull/19889))
 
-### Deprecated
+- ### Deprecated
 - Deprecated existing constructors in ThreadPoolStats.Stats in favor of the new Builder ([#19317](https://github.com/opensearch-project/OpenSearch/pull/19317))
 - Deprecated existing constructors in IndexingStats.Stats in favor of the new Builder ([#19306](https://github.com/opensearch-project/OpenSearch/pull/19306))
+- Deprecated existing constructors in RefreshStats in favor of the new Builder ([#19835](https://github.com/opensearch-project/OpenSearch/pull/19835))
+- Deprecated existing constructors in DocStats and StoreStats in favor of the new Builder ([#19863](https://github.com/opensearch-project/OpenSearch/pull/19863))
 
 ### Removed
 

build.gradle

@@ -55,7 +55,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.25.0" apply false
+  id "com.diffplug.spotless" version "8.0.0" apply false
   id "org.gradle.test-retry" version "1.6.2" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
@@ -271,10 +271,7 @@ allprojects {
       memoryMaximumSize = project.property('options.forkOptions.memoryMaximumSize')
     }
 
-    // See please https://bugs.openjdk.java.net/browse/JDK-8209058
-    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_11) {
-      compile.options.compilerArgs << '-Werror'
-    }
+    compile.options.compilerArgs << '-Werror'
     compile.options.compilerArgs << '-Xlint:auxiliaryclass'
     compile.options.compilerArgs << '-Xlint:cast'
     compile.options.compilerArgs << '-Xlint:classfile'

buildSrc/src/integTest/groovy/org/opensearch/gradle/DistributionDownloadPluginFuncTest.groovy

@@ -30,6 +30,7 @@
 package org.opensearch.gradle
 
 
+import org.opensearch.gradle.Architecture
 import org.opensearch.gradle.fixtures.AbstractGradleFuncTest
 import org.opensearch.gradle.transform.SymbolicLinkPreservingUntarTransform
 import org.gradle.testkit.runner.TaskOutcome
@@ -65,6 +66,7 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
         given:
         def version = VersionProperties.getOpenSearch()
         def platform = OpenSearchDistribution.Platform.LINUX
+        def arch = Architecture.current().name().toLowerCase()
 
         buildFile << applyPluginAndSetupDistro(version, platform)
         buildFile << """
@@ -83,14 +85,15 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
         then:
         result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
-        result.output.count("Unpacking opensearch-${version}-linux-x64.tar.gz " +
+        result.output.count("Unpacking opensearch-${version}-linux-${arch}.tar.gz " +
             "using SymbolicLinkPreservingUntarTransform.") == 0
     }
 
     def "transforms are reused across projects"() {
         given:
         def version = VersionProperties.getOpenSearch()
         def platform = OpenSearchDistribution.Platform.LINUX
+        def arch = Architecture.current().name().toLowerCase()
 
         3.times {
             testProjectDir.newFolder("sub-$it")
@@ -122,7 +125,7 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
         then:
         result.tasks.size() == 3
-        result.output.count("Unpacking opensearch-${version}-linux-x64.tar.gz " +
+        result.output.count("Unpacking opensearch-${version}-linux-${arch}.tar.gz " +
                 "using SymbolicLinkPreservingUntarTransform.") == 1
     }
 

buildSrc/src/integTest/groovy/org/opensearch/gradle/fixtures/DistributionDownloadFixture.groovy

@@ -30,6 +30,7 @@
 package org.opensearch.gradle.fixtures
 
 
+import org.opensearch.gradle.Architecture
 import org.opensearch.gradle.OpenSearchDistribution
 import org.opensearch.gradle.Version
 import org.opensearch.gradle.VersionProperties
@@ -66,11 +67,12 @@ class DistributionDownloadFixture {
     private static String urlPath(String version, OpenSearchDistribution.Platform platform) {
         String fileType = ((platform == OpenSearchDistribution.Platform.LINUX ||
                 platform == OpenSearchDistribution.Platform.DARWIN)) ? "tar.gz" : "zip"
+        String arch = Architecture.current().name().toLowerCase()
         if (Version.fromString(version).onOrAfter(Version.fromString("1.0.0"))) {
             if (version.contains("SNAPSHOT")) {
-                return "/snapshots/core/opensearch/${version}/opensearch-min-${version}-${platform}-x64-latest.$fileType"
+                return "/snapshots/core/opensearch/${version}/opensearch-min-${version}-${platform}-${arch}-latest.$fileType"
             }
-            return "/releases/core/opensearch/${version}/opensearch-min-${version}-${platform}-x64.$fileType"
+            return "/releases/core/opensearch/${version}/opensearch-min-${version}-${platform}-${arch}.$fileType"
         } else {
             return "/downloads/elasticsearch/elasticsearch-oss-${version}-${platform}-x86_64.$fileType"
         }

buildSrc/src/integTest/groovy/org/opensearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -29,6 +29,7 @@
 
 package org.opensearch.gradle.internal
 
+import org.opensearch.gradle.Architecture
 import org.opensearch.gradle.VersionProperties
 import org.opensearch.gradle.fixtures.AbstractGradleFuncTest
 import org.gradle.testkit.runner.GradleRunner
@@ -59,7 +60,8 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
     def "resolves current version from local build"() {
         given:
         internalBuild()
-        localDistroSetup()
+        def archive = archiveTask()
+        localDistroSetup(archive)
         def distroVersion = VersionProperties.getOpenSearch()
         buildFile << """
             apply plugin: 'opensearch.internal-distribution-download'
@@ -82,7 +84,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build()
 
         then:
-        result.task(":distribution:archives:linux-tar:buildExpanded").outcome == TaskOutcome.SUCCESS
+        result.task(":distribution:archives:${archive}:buildExpanded").outcome == TaskOutcome.SUCCESS
         result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
         assertExtractedDistroIsCreated("build/distro", 'current-marker.txt')
     }
@@ -150,42 +152,47 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         settingsFile << """
         include ':distribution:bwc:minor'
         """
+        def archive = archiveTask()
         def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", "minor")
         new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=minor"
         new File(bwcSubProjectFolder, 'build.gradle') << """
             apply plugin:'base'
 
             // packed distro
-            configurations.create("linux-tar")
+            configurations.create("${archive}")
             tasks.register("buildBwcTask", Tar) {
                 from('bwc-marker.txt')
                 archiveExtension = "tar.gz"
                 compression = Compression.GZIP
             }
             artifacts {
-                it.add("linux-tar", buildBwcTask)
+                it.add("${archive}", buildBwcTask)
             }
 
             // expanded distro
-            configurations.create("expanded-linux-tar")
+            configurations.create("expanded-${archive}")
             def expandedTask = tasks.register("buildBwcExpandedTask", Copy) {
                 from('bwc-marker.txt')
                 into('build/install/opensearch-distro')
             }
             artifacts {
-                it.add("expanded-linux-tar", file('build/install')) {
+                it.add("expanded-${archive}", file('build/install')) {
                     builtBy expandedTask
                     type = 'directory'
                 }
             }
         """
     }
 
-    private void localDistroSetup() {
+    private String archiveTask() {
+        return Architecture.current() == Architecture.X64 ? "linux-tar" : "linux-${Architecture.current().name().toLowerCase()}-tar"; 
+    }
+
+    private void localDistroSetup(def archive) {
         settingsFile << """
-        include ":distribution:archives:linux-tar"
+        include ":distribution:archives:${archive}"
         """
-        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", "linux-tar")
+        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", "${archive}")
         new File(bwcSubProjectFolder, 'current-marker.txt') << "current"
         new File(bwcSubProjectFolder, 'build.gradle') << """
             import org.gradle.api.internal.artifacts.ArtifactAttributes;

buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java

@@ -51,7 +51,7 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
     public TaskProvider<? extends Task> createTask(Project project) {
         project.getPlugins().apply(CompileOnlyResolvePlugin.class);
         project.getConfigurations().create("forbiddenApisCliJar");
-        project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.8");
+        project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.10");
 
         Configuration jdkJarHellConfig = project.getConfigurations().create(JDK_JAR_HELL_CONFIG_NAME);
         if (BuildParams.isInternal() && project.getPath().equals(":libs:opensearch-core") == false) {

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -77,9 +77,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "24.0.2+12";
+    private static final String SYSTEM_JDK_VERSION = "25.0.1+8";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "24.0.2+12";
+    private static final String GRADLE_JDK_VERSION = "25.0.1+8";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution