Merge branch 'main' into deven009/fix/wildcard-escaped-backslash-19719

Signed-off-by: Deven009 <[email protected]>

Author: Deven009

.ci/bwcVersions

@@ -52,3 +52,4 @@ BWC_VERSION:
   - "3.2.0"
   - "3.3.0"
   - "3.3.1"
+  - "3.3.2"

.github/workflows/codeql-analysis.yml

@@ -43,7 +43,7 @@ jobs:
         distribution: temurin
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -54,7 +54,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      uses: github/codeql-action/autobuild@v4
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,4 +68,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v4

.github/workflows/wrapper.yml

@@ -8,4 +8,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
-      - uses: gradle/actions/wrapper-validation@v4
+      - uses: gradle/actions/wrapper-validation@v5

CHANGELOG.md

@@ -11,9 +11,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Use Lucene `pack` method for `half_float` and `usigned_long` when using `ApproximatePointRangeQuery`.
 - Add a mapper for context aware segments grouping criteria ([#19233](https://github.com/opensearch-project/OpenSearch/pull/19233))
 - Return full error for GRPC error response ([#19568](https://github.com/opensearch-project/OpenSearch/pull/19568))
+- Add support for repository with Server side encryption enabled and client side encryption as well based on a flag. ([#19630)](https://github.com/opensearch-project/OpenSearch/pull/19630))
 - Add pluggable gRPC interceptors with explicit ordering([#19005](https://github.com/opensearch-project/OpenSearch/pull/19005))
+- Add BindableServices extension point to transport-grpc-spi ([#19304](https://github.com/opensearch-project/OpenSearch/pull/19304))
 - Add metrics for the merged segment warmer feature ([#18929](https://github.com/opensearch-project/OpenSearch/pull/18929))
 - Add pointer based lag metric in pull-based ingestion ([#19635](https://github.com/opensearch-project/OpenSearch/pull/19635))
+- Introduced internal API for retrieving metadata about requested indices from transport actions  ([#18523](https://github.com/opensearch-project/OpenSearch/pull/18523))
+- Add cluster defaults for merge autoThrottle, maxMergeThreads, and maxMergeCount; Add segment size filter to the merged segment warmer ([#19629](https://github.com/opensearch-project/OpenSearch/pull/19629))
+- Add build-tooling to run in FIPS environment ([#18921](https://github.com/opensearch-project/OpenSearch/pull/18921))
+- Add SMILE/CBOR/YAML document format support to Bulk GRPC endpoint ([#19744](https://github.com/opensearch-project/OpenSearch/pull/19744))
 
 ### Changed
 - Faster `terms` query creation for `keyword` field with index and docValues enabled ([#19350](https://github.com/opensearch-project/OpenSearch/pull/19350))
@@ -23,18 +29,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remove MultiCollectorWrapper and use MultiCollector in Lucene instead ([#19595](https://github.com/opensearch-project/OpenSearch/pull/19595))
 - Change implementation for `percentiles` aggregation for latency improvement ([#19648](https://github.com/opensearch-project/OpenSearch/pull/19648))
 - Wrap checked exceptions in painless.DefBootstrap to support JDK-25 ([#19706](https://github.com/opensearch-project/OpenSearch/pull/19706))
-- Refactor the ThreadPoolStats.Stats class to use the Builder pattern instead of constructors ([#19306](https://github.com/opensearch-project/OpenSearch/pull/19306))
+- Refactor the ThreadPoolStats.Stats class to use the Builder pattern instead of constructors ([#19317](https://github.com/opensearch-project/OpenSearch/pull/19317))
 - Refactor the IndexingStats.Stats class to use the Builder pattern instead of constructors ([#19306](https://github.com/opensearch-project/OpenSearch/pull/19306))
+- Remove FeatureFlag.MERGED_SEGMENT_WARMER_EXPERIMENTAL_FLAG. ([#19715](https://github.com/opensearch-project/OpenSearch/pull/19715))
+- Change the default value of doc_values in WildcardFieldMapper to true. ([#19796](https://github.com/opensearch-project/OpenSearch/pull/19796))
+- Make Engine#loadHistoryUUID() protected and Origin#isFromTranslog() public ([#19753](https://github.com/opensearch-project/OpenSearch/pull/19752))
 
 ### Fixed
 - Fix Allocation and Rebalance Constraints of WeightFunction are incorrectly reset ([#19012](https://github.com/opensearch-project/OpenSearch/pull/19012))
 - Fix flaky test FieldDataLoadingIT.testIndicesFieldDataCacheSizeSetting ([#19571](https://github.com/opensearch-project/OpenSearch/pull/19571))
+- Fix case-insensitive wildcard + aggregation query crash ([#19489](https://github.com/opensearch-project/OpenSearch/pull/19489))
 - Avoid primary shard failure caused by merged segment warmer exceptions ([#19436](https://github.com/opensearch-project/OpenSearch/pull/19436))
 - Fix pull-based ingestion out-of-bounds offset scenarios and remove persisted offsets ([#19607](https://github.com/opensearch-project/OpenSearch/pull/19607))
 - Fix issue with updating core with a patch number other than 0 ([#19377](https://github.com/opensearch-project/OpenSearch/pull/19377))
 - [Java Agent] Allow JRT protocol URLs in protection domain extraction ([#19683](https://github.com/opensearch-project/OpenSearch/pull/19683))
 - Fix potential concurrent modification exception when updating allocation filters ([#19701])(https://github.com/opensearch-project/OpenSearch/pull/19701))
 - Fix wildcard query with escaped backslash followed by wildcard character ([#19719](https://github.com/opensearch-project/OpenSearch/pull/19719))
+- Fix file-based ingestion consumer to handle start point beyond max line number([#19757])(https://github.com/opensearch-project/OpenSearch/pull/19757))
+- Fix IndexOutOfBoundsException when running include/exclude on non-existent prefix in terms aggregations ([#19637](https://github.com/opensearch-project/OpenSearch/pull/19637))
+- Fixed assertion unsafe use of ClusterService.state() in ResourceUsageCollectorService ([#19775])(https://github.com/opensearch-project/OpenSearch/pull/19775))
+- Add S3Repository.LEGACY_MD5_CHECKSUM_CALCULATION to list of repository-s3 settings ([#19788](https://github.com/opensearch-project/OpenSearch/pull/19788))
 
 ### Dependencies
 - Update to Gradle 9.1 ([#19575](https://github.com/opensearch-project/OpenSearch/pull/19575))
@@ -49,9 +63,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.avro:avro` from 1.12.0 to 1.12.1 ([#19692](https://github.com/opensearch-project/OpenSearch/pull/19692))
 - Bump `com.github.spotbugs:spotbugs-annotations` from 4.9.6 to 4.9.8 ([#19691](https://github.com/opensearch-project/OpenSearch/pull/19691))
 - Bump `stefanzweifel/git-auto-commit-action` from 6 to 7 ([#19689](https://github.com/opensearch-project/OpenSearch/pull/19689))
+- Bump ch.qos.logback modules from 1.5.18 to 1.5.20 in HDFS test fixture ([#19763](https://github.com/opensearch-project/OpenSearch/pull/19763))
+- Bump `github/codeql-action` from 3 to 4 ([#19785](https://github.com/opensearch-project/OpenSearch/pull/19785))
+- Bump `gradle/actions` from 4 to 5 ([#19781](https://github.com/opensearch-project/OpenSearch/pull/19781))
 
 ### Deprecated
-- Deprecated existing constructors in ThreadPoolStats.Stats in favor of the new Builder ([#19306](https://github.com/opensearch-project/OpenSearch/pull/19306))
+- Deprecated existing constructors in ThreadPoolStats.Stats in favor of the new Builder ([#19317](https://github.com/opensearch-project/OpenSearch/pull/19317))
 - Deprecated existing constructors in IndexingStats.Stats in favor of the new Builder ([#19306](https://github.com/opensearch-project/OpenSearch/pull/19306))
 
 ### Removed

build.gradle

@@ -71,6 +71,11 @@ apply from: 'gradle/run.gradle'
 apply from: 'gradle/missing-javadoc.gradle'
 apply from: 'gradle/code-coverage.gradle'
 
+// Apply FIPS configuration to all projects
+allprojects {
+  apply from: "$rootDir/gradle/fips.gradle"
+}
+
 // common maven publishing configuration
 allprojects {
   group = 'org.opensearch'
@@ -421,8 +426,12 @@ gradle.projectsEvaluated {
           dependsOn(project(':libs:agent-sm:agent').prepareAgent)
           jvmArgs += ["-javaagent:" + project(':libs:agent-sm:agent').jar.archiveFile.get()]
         }
-        if (BuildParams.inFipsJvm) {
-          task.jvmArgs += ["-Dorg.bouncycastle.fips.approved_only=true"]
+        if (BuildParams.isInFipsJvm()) {
+          def fipsSecurityFile = project.rootProject.file('distribution/src/config/fips_java.security')
+          task.jvmArgs += [
+              "-Dorg.bouncycastle.fips.approved_only=true",
+              "-Djava.security.properties=${fipsSecurityFile}"
+          ]
         }
       }
     }
@@ -693,14 +702,6 @@ allprojects {
   plugins.withId('lifecycle-base') {
     checkPart1.configure { dependsOn 'check' }
   }
-
-  plugins.withId('opensearch.testclusters') {
-    testClusters.configureEach {
-      if (BuildParams.inFipsJvm) {
-        keystorePassword 'notarealpasswordphrase'
-      }
-    }
-  }
 }
 
 subprojects {

buildSrc/build.gradle

@@ -133,6 +133,14 @@ dependencies {
   integTestImplementation('org.spockframework:spock-core:2.4-M6-groovy-4.0') {
     exclude module: "groovy"
   }
+  // Integration with JUnit 4 features for Spock 2+ (which internally uses JUnit Platform -
+  // part of JUnit 5) - https://spockframework.org/spock/docs/2.4-M6/modules.html
+  integTestImplementation('org.spockframework:spock-junit4:2.4-M6-groovy-4.0') {
+    exclude module: "groovy"
+  }
+  integTestImplementation(platform("org.junit:junit-bom:${props.getProperty('junit5')}"))
+  integTestRuntimeOnly("org.junit.platform:junit-platform-launcher")
+  integTestRuntimeOnly("org.junit.vintage:junit-vintage-engine")
 }
 
 configurations.all {
@@ -185,7 +193,7 @@ if (project != rootProject) {
   disableTasks('forbiddenApisMain', 'forbiddenApisTest', 'forbiddenApisIntegTest', 'forbiddenApisTestFixtures')
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
-  if (org.opensearch.gradle.info.BuildParams.inFipsJvm) {
+  if (org.opensearch.gradle.info.BuildParams.isInFipsJvm()) {
     // We don't support running gradle with a JVM that is in FIPS 140 mode, so we don't test it.
     // WaitForHttpResourceTests tests would fail as they use JKS/PKCS12 keystores
     test.enabled = false
@@ -253,9 +261,10 @@ if (project != rootProject) {
   }
 
   tasks.register("integTest", Test) {
+    useJUnitPlatform()
     inputs.dir(file("src/testKit")).withPropertyName("testkit dir").withPathSensitivity(PathSensitivity.RELATIVE)
     systemProperty 'test.version_under_test', version
-    onlyIf { org.opensearch.gradle.info.BuildParams.inFipsJvm == false }
+    onlyIf { org.opensearch.gradle.info.BuildParams.isInFipsJvm() == false }
     maxParallelForks = System.getProperty('tests.jvms', org.opensearch.gradle.info.BuildParams.defaultParallel.toString()) as Integer
     testClassesDirs = sourceSets.integTest.output.classesDirs
     classpath = sourceSets.integTest.runtimeClasspath

buildSrc/src/integTest/groovy/org/opensearch/gradle/DistributionDownloadPluginFuncTest.groovy

@@ -56,8 +56,8 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
         where:
         version                              | platform                                   | distType
         VersionProperties.getOpenSearch()    | OpenSearchDistribution.Platform.LINUX      | "current"
-        "8.1.0-SNAPSHOT"                     | OpenSearchDistribution.Platform.LINUX   | "bwc"
-        "7.0.0"                              | OpenSearchDistribution.Platform.WINDOWS | "released"
+        "2.19.0-SNAPSHOT"                    | OpenSearchDistribution.Platform.LINUX   | "bwc"
+        "3.3.0"                              | OpenSearchDistribution.Platform.WINDOWS | "released"
     }
 
 
@@ -72,7 +72,8 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
         """
 
         when:
-        def runner = gradleRunner('clean', 'setupDistro', '-i')
+        def customGradleUserHome = testProjectDir.newFolder().absolutePath;
+        def runner = gradleRunner('clean', 'setupDistro', '-i', '-g', customGradleUserHome)
         def result = withMockedDistributionDownload(version, platform, runner) {
             // initial run
             build()
@@ -82,7 +83,8 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
         then:
         result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
-        assertOutputContains(result.output, "Skipping ${SymbolicLinkPreservingUntarTransform.class.simpleName}")
+        result.output.count("Unpacking opensearch-${version}-linux-x64.tar.gz " +
+            "using SymbolicLinkPreservingUntarTransform.") == 0
     }
 
     def "transforms are reused across projects"() {
@@ -91,6 +93,7 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
         def platform = OpenSearchDistribution.Platform.LINUX
 
         3.times {
+            testProjectDir.newFolder("sub-$it")
             settingsFile << """
                 include ':sub-$it'
             """

buildSrc/src/integTest/groovy/org/opensearch/gradle/JdkDownloadPluginFuncTest.groovy

@@ -108,6 +108,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform)
         def mockedContent = filebytes(jdkVendor, platform)
         3.times {
+            testProjectDir.newFolder("sub-$it")
             settingsFile << """
                 include ':sub-$it'
             """
@@ -193,7 +194,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         }
 
         then:
-        assertOutputContains(result.output, "Skipping $transformType")
+        result.output.count("Unpacking ${platform}-12.0.2-x64.tar.gz using ${transformType}.") == 0
 
         where:
         platform  | transformType
@@ -246,6 +247,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
                 p.repositories.all { repo ->
                     if(repo.name == "jdk_repo_${jdkVendor}_${jdkVersion}") {
                       repo.setUrl('${server.baseUrl()}')
+                      repo.setAllowInsecureProtocol(true)
                     }
                 }
            }"""

buildSrc/src/integTest/groovy/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy

@@ -98,6 +98,9 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
         given:
         file('someFile.txt') << "some content"
 
+        testProjectDir.newFolder("consumer")
+        testProjectDir.newFolder("producer-tar")
+
         settingsFile << """
             include ':consumer'
             include ':producer-tar'

buildSrc/src/integTest/groovy/org/opensearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -122,6 +122,8 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         internalBuild()
         bwcMinorProjectSetup()
         buildFile << """
+            import org.opensearch.gradle.JavaPackageType
+
             apply plugin: 'opensearch.internal-distribution-download'
 
             opensearch_distributions {
@@ -130,7 +132,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
                   type = "archive"
                   platform = "linux"
                   architecture = Architecture.current();
-                  bundledJdk = false
+                  bundledJdk = JavaPackageType.NONE
               }
             }
             tasks.register("createExtractedTestDistro") {