Merge branch 'main' into main

Signed-off-by: Tommy Shao <[email protected]>
opensearch-project · Jan 6, 2025 · 4a659c0 · 4a659c0
2 parents 736696c + e73ffdf
commit 4a659c0
Show file tree

Hide file tree

Showing 228 changed files with 5,485 additions and 5,275 deletions.
diff --git a/.github/benchmark-configs.json b/.github/benchmark-configs.json
@@ -239,5 +239,22 @@
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
+  },
+  "id_15": {
+    "description": "Search only test-procedure for big5, uses lucene-10 index snapshot to restore the data for OS-3.0.0",
+    "supported_major_versions": ["3"],
+    "cluster-benchmark-configs": {
+      "SINGLE_NODE_CLUSTER": "true",
+      "MIN_DISTRIBUTION": "true",
+      "TEST_WORKLOAD": "big5",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
+      "CAPTURE_NODE_STAT": "true",
+      "TEST_PROCEDURE": "restore-from-snapshot"
+    },
+    "cluster_configuration": {
+      "size": "Single-Node",
+      "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   }
 }
diff --git a/.github/workflows/benchmark-pull-request.yml b/.github/workflows/benchmark-pull-request.yml
@@ -4,7 +4,10 @@ on:
     types: [created]
 jobs:
   run-performance-benchmark-on-pull-request:
-    if: ${{ (github.event.issue.pull_request) && (contains(github.event.comment.body, '"run-benchmark-test"')) }}
+    if: |
+      github.repository == 'opensearch-project/OpenSearch' &&
+      github.event.issue.pull_request &&
+      contains(github.event.comment.body, '"run-benchmark-test"')
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -111,7 +114,7 @@ jobs:
         uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          result-encoding: string
+          result-encoding: json
           script: |
             // Get the collaborators - filtered to maintainer permissions
             const maintainersResponse = await github.request('GET /repos/{owner}/{repo}/collaborators', {
@@ -121,12 +124,12 @@ jobs:
               affiliation: 'all',
               per_page: 100
               });
-            return maintainersResponse.data.map(item => item.login).join(', ');
+            return maintainersResponse.data.map(item => item.login);
       - uses: trstringer/manual-approval@v1
-        if: (!contains(steps.get_approvers.outputs.result, github.event.comment.user.login))
+        if: ${{ !contains(fromJSON(steps.get_approvers.outputs.result), github.event.comment.user.login) }}
         with:
           secret: ${{ github.TOKEN }}
-          approvers: ${{ steps.get_approvers.outputs.result }}
+          approvers: ${{ join(fromJSON(steps.get_approvers.outputs.result), ', ') }}
           minimum-approvals: 1
           issue-title: 'Request to approve/deny benchmark run for PR #${{ env.PR_NUMBER }}'
           issue-body: "Please approve or deny the benchmark run for PR #${{ env.PR_NUMBER }}"

diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml
diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml
@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: lychee Link Checker
         id: lychee
-        uses: lycheeverse/lychee-action@v2.1.0
+        uses: lycheeverse/lychee-action@v2.2.0
         with:
           args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes
           fail: true

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -18,33 +18,52 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support for keyword fields in star-tree index ([#16233](https://github.com/opensearch-project/OpenSearch/pull/16233))
 - Add a flag in QueryShardContext to differentiate inner hit query ([#16600](https://github.com/opensearch-project/OpenSearch/pull/16600))
 - Add vertical scaling and SoftReference for snapshot repository data cache ([#16489](https://github.com/opensearch-project/OpenSearch/pull/16489))
+- [Workload Management] Add Workload Management IT ([#16359](https://github.com/opensearch-project/OpenSearch/pull/16359))
 - Support prefix list for remote repository attributes([#16271](https://github.com/opensearch-project/OpenSearch/pull/16271))
 - Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)).
 - Add stats for remote publication failure and move download failure stats to remote methods([#16682](https://github.com/opensearch-project/OpenSearch/pull/16682/))
+- Update script supports java.lang.String.sha1() and java.lang.String.sha256() methods ([#16923](https://github.com/opensearch-project/OpenSearch/pull/16923))
+- Added a precaution to handle extreme date values during sorting to prevent `arithmetic_exception: long overflow` ([#16812](https://github.com/opensearch-project/OpenSearch/pull/16812)).
+- Add search replica stats to segment replication stats API ([#16678](https://github.com/opensearch-project/OpenSearch/pull/16678))
+- Introduce a setting to disable download of full cluster state from remote on term mismatch([#16798](https://github.com/opensearch-project/OpenSearch/pull/16798/))
+- Added ability to retrieve value from DocValues in a flat_object filed([#16802](https://github.com/opensearch-project/OpenSearch/pull/16802))
+- Introduce framework for auxiliary transports and an experimental gRPC transport plugin ([#16534](https://github.com/opensearch-project/OpenSearch/pull/16534))
+- Changes to support IP field in star tree indexing([#16641](https://github.com/opensearch-project/OpenSearch/pull/16641/))
+- Support object fields in star-tree index([#16728](https://github.com/opensearch-project/OpenSearch/pull/16728/))
 
 ### Dependencies
 - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))
 - Bump `google-auth-library-oauth2-http` from 1.7.0 to 1.29.0 in /plugins/repository-gcs ([#16520](https://github.com/opensearch-project/OpenSearch/pull/16520))
-- Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521))
+- Bump `com.azure:azure-storage-common` from 12.25.1 to 12.28.0 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521), [#16808](https://github.com/opensearch-project/OpenSearch/pull/16808))
 - Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241105-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548), [#16613](https://github.com/opensearch-project/OpenSearch/pull/16613))
 - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501))
 - Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.0 to 3.4.1 ([#16550](https://github.com/opensearch-project/OpenSearch/pull/16550))
-- Bump `org.apache.xmlbeans:xmlbeans` from 5.2.1 to 5.2.2 ([#16612](https://github.com/opensearch-project/OpenSearch/pull/16612))
-- Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 9.46 ([#16611](https://github.com/opensearch-project/OpenSearch/pull/16611))
-- Bump `lycheeverse/lychee-action` from 2.0.2 to 2.1.0 ([#16610](https://github.com/opensearch-project/OpenSearch/pull/16610))
+- Bump `org.apache.xmlbeans:xmlbeans` from 5.2.1 to 5.3.0 ([#16612](https://github.com/opensearch-project/OpenSearch/pull/16612), [#16854](https://github.com/opensearch-project/OpenSearch/pull/16854))
+- Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 9.47 ([#16611](https://github.com/opensearch-project/OpenSearch/pull/16611), [#16807](https://github.com/opensearch-project/OpenSearch/pull/16807))
+- Bump `lycheeverse/lychee-action` from 2.0.2 to 2.2.0 ([#16610](https://github.com/opensearch-project/OpenSearch/pull/16610), [#16897](https://github.com/opensearch-project/OpenSearch/pull/16897))
 - Bump `me.champeau.gradle.japicmp` from 0.4.4 to 0.4.5 ([#16614](https://github.com/opensearch-project/OpenSearch/pull/16614))
 - Bump `mockito` from 5.14.1 to 5.14.2, `objenesis` from 3.2 to 3.3 and `bytebuddy` from 1.15.4 to 1.15.10 ([#16655](https://github.com/opensearch-project/OpenSearch/pull/16655))
 - Bump `Netty` from 4.1.114.Final to 4.1.115.Final ([#16661](https://github.com/opensearch-project/OpenSearch/pull/16661))
 - Bump `org.xerial.snappy:snappy-java` from 1.1.10.6 to 1.1.10.7 ([#16665](https://github.com/opensearch-project/OpenSearch/pull/16665))
 - Bump `codecov/codecov-action` from 4 to 5 ([#16667](https://github.com/opensearch-project/OpenSearch/pull/16667))
-- Bump `org.apache.logging.log4j:log4j-core` from 2.24.1 to 2.24.2 ([#16718](https://github.com/opensearch-project/OpenSearch/pull/16718))
+- Bump `org.apache.logging.log4j:log4j-core` from 2.24.1 to 2.24.3 ([#16718](https://github.com/opensearch-project/OpenSearch/pull/16718), [#16858](https://github.com/opensearch-project/OpenSearch/pull/16858))
 - Bump `jackson` from 2.17.2 to 2.18.2 ([#16733](https://github.com/opensearch-project/OpenSearch/pull/16733))
-- Bump `ch.qos.logback:logback-classic` from 1.2.13 to 1.5.12 ([#16716](https://github.com/opensearch-project/OpenSearch/pull/16716))
+- Bump `ch.qos.logback:logback-classic` from 1.2.13 to 1.5.15 ([#16716](https://github.com/opensearch-project/OpenSearch/pull/16716), [#16898](https://github.com/opensearch-project/OpenSearch/pull/16898))
 - Bump `com.azure:azure-identity` from 1.13.2 to 1.14.2 ([#16778](https://github.com/opensearch-project/OpenSearch/pull/16778))
+- Bump Apache Lucene from 9.12.0 to 9.12.1 ([#16846](https://github.com/opensearch-project/OpenSearch/pull/16846))
+- Bump `com.gradle.develocity` from 3.18.2 to 3.19 ([#16855](https://github.com/opensearch-project/OpenSearch/pull/16855))
+- Bump `org.jline:jline` from 3.27.1 to 3.28.0 ([#16857](https://github.com/opensearch-project/OpenSearch/pull/16857))
+- Bump `com.azure:azure-core` from 1.51.0 to 1.54.1 ([#16856](https://github.com/opensearch-project/OpenSearch/pull/16856))
+- Bump `com.nimbusds:oauth2-oidc-sdk` from 11.19.1 to 11.20.1 ([#16895](https://github.com/opensearch-project/OpenSearch/pull/16895))
+- Bump `com.netflix.nebula.ospackage-base` from 11.10.0 to 11.10.1 ([#16896](https://github.com/opensearch-project/OpenSearch/pull/16896))
+- Bump `com.microsoft.azure:msal4j` from 1.17.2 to 1.18.0 ([#16918](https://github.com/opensearch-project/OpenSearch/pull/16918))
+- Bump `org.apache.commons:commons-text` from 1.12.0 to 1.13.0 ([#16919](https://github.com/opensearch-project/OpenSearch/pull/16919))
+- Bump `ch.qos.logback:logback-core` from 1.5.12 to 1.5.16 ([#16951](https://github.com/opensearch-project/OpenSearch/pull/16951))
 
 ### Changed
 - Indexed IP field supports `terms_query` with more than 1025 IP masks [#16391](https://github.com/opensearch-project/OpenSearch/pull/16391)
 - Make entries for dependencies from server/build.gradle to gradle version catalog ([#16707](https://github.com/opensearch-project/OpenSearch/pull/16707))
+- Allow extended plugins to be optional ([#16909](https://github.com/opensearch-project/OpenSearch/pull/16909))
 
 ### Deprecated
 - Performing update operation with default pipeline or final pipeline is deprecated ([#16712](https://github.com/opensearch-project/OpenSearch/pull/16712))
@@ -64,7 +83,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Tiered Caching] Fix bug in cache stats API ([#16560](https://github.com/opensearch-project/OpenSearch/pull/16560))
 - Bound the size of cache in deprecation logger ([16702](https://github.com/opensearch-project/OpenSearch/issues/16702))
 - Ensure consistency of system flag on IndexMetadata after diff is applied ([#16644](https://github.com/opensearch-project/OpenSearch/pull/16644))
-- Fixing the bug to prevent updating the index.knn setting during restore snapshot(#)[]
+- Skip remote-repositories validations for node-joins when RepositoriesService is not in sync with cluster-state ([#16763](https://github.com/opensearch-project/OpenSearch/pull/16763))
+- Fix _list/shards API failing when closed indices are present ([#16606](https://github.com/opensearch-project/OpenSearch/pull/16606))
+- Fix remote shards balance ([#15335](https://github.com/opensearch-project/OpenSearch/pull/15335))
+- Fixing the bug to prevent updating the index.knn setting during restore snapshot(#16957)[https://github.com/opensearch-project/OpenSearch/pull/16957]
+
 
 ### Security
 

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java b/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java
@@ -105,16 +105,15 @@ public DockerAvailability getDockerAvailability() {
             Result lastResult = null;
             Version version = null;
             boolean isVersionHighEnough = false;
-            boolean isComposeAvailable = false;
-            boolean isComposeV2Available = false;
+            DockerComposeAvailability dockerComposeAvailability = null;
 
             // Check if the Docker binary exists
             final Optional<String> dockerBinary = getDockerPath();
             if (isExcludedOs() == false && dockerBinary.isPresent()) {
                 dockerPath = dockerBinary.get();
 
                 // Since we use a multi-stage Docker build, check the Docker version meets minimum requirement
-                lastResult = runCommand(dockerPath, "version", "--format", "{{.Server.Version}}");
+                lastResult = runCommand(execOperations, dockerPath, "version", "--format", "{{.Server.Version}}");
 
                 if (lastResult.isSuccess()) {
                     version = Version.fromString(lastResult.stdout.trim(), Version.Mode.RELAXED);
@@ -123,15 +122,11 @@ public DockerAvailability getDockerAvailability() {
 
                     if (isVersionHighEnough) {
                         // Check that we can execute a privileged command
-                        lastResult = runCommand(dockerPath, "images");
-
+                        lastResult = runCommand(execOperations, dockerPath, "images");
                         // If docker all checks out, see if docker-compose is available and working
-                        Optional<String> composePath = getDockerComposePath();
-                        if (lastResult.isSuccess() && composePath.isPresent()) {
-                            isComposeAvailable = runCommand(composePath.get(), "version").isSuccess();
+                        if (lastResult.isSuccess()) {
+                            dockerComposeAvailability = DockerComposeAvailability.detect(execOperations, dockerPath).orElse(null);
                         }
-
-                        isComposeV2Available = runCommand(dockerPath, "compose", "version").isSuccess();
                     }
                 }
             }
@@ -140,8 +135,7 @@ public DockerAvailability getDockerAvailability() {
 
             this.dockerAvailability = new DockerAvailability(
                 isAvailable,
-                isComposeAvailable,
-                isComposeV2Available,
+                dockerComposeAvailability,
                 isVersionHighEnough,
                 dockerPath,
                 version,
@@ -291,17 +285,6 @@ private Optional<String> getDockerPath() {
         return Arrays.asList(DOCKER_BINARIES).stream().filter(path -> new File(path).exists()).findFirst();
     }
 
-    /**
-     * Searches the entries in {@link #DOCKER_COMPOSE_BINARIES} for the Docker Compose CLI. This method does
-     * not check whether the installation appears usable, see {@link #getDockerAvailability()} instead.
-     *
-     * @return the path to a CLI, if available.
-     */
-    private Optional<String> getDockerComposePath() {
-        // Check if the Docker binary exists
-        return Arrays.asList(DOCKER_COMPOSE_BINARIES).stream().filter(path -> new File(path).exists()).findFirst();
-    }
-
     private void throwDockerRequiredException(final String message) {
         throwDockerRequiredException(message, null);
     }
@@ -321,7 +304,7 @@ private void throwDockerRequiredException(final String message, Exception e) {
      * while running the command, or the process was killed after reaching the 10s timeout,
      * then the exit code will be -1.
      */
-    private Result runCommand(String... args) {
+    private static Result runCommand(ExecOperations execOperations, String... args) {
         if (args.length == 0) {
             throw new IllegalArgumentException("Cannot execute with no command");
         }
@@ -356,14 +339,9 @@ public static class DockerAvailability {
         public final boolean isAvailable;
 
         /**
-         * True if docker-compose is available.
+         * Non-null if docker-compose v1 or v2 is available.
          */
-        public final boolean isComposeAvailable;
-
-        /**
-         * True if docker compose is available.
-         */
-        public final boolean isComposeV2Available;
+        public final DockerComposeAvailability dockerComposeAvailability;
 
         /**
          * True if the installed Docker version is &gt;= 17.05
@@ -387,23 +365,70 @@ public static class DockerAvailability {
 
         DockerAvailability(
             boolean isAvailable,
-            boolean isComposeAvailable,
-            boolean isComposeV2Available,
+            DockerComposeAvailability dockerComposeAvailability,
             boolean isVersionHighEnough,
             String path,
             Version version,
             Result lastCommand
         ) {
             this.isAvailable = isAvailable;
-            this.isComposeAvailable = isComposeAvailable;
-            this.isComposeV2Available = isComposeV2Available;
+            this.dockerComposeAvailability = dockerComposeAvailability;
             this.isVersionHighEnough = isVersionHighEnough;
             this.path = path;
             this.version = version;
             this.lastCommand = lastCommand;
         }
+
+        public boolean isDockerComposeAvailable() {
+            return dockerComposeAvailability != null;
+        }
+    }
+
+    /**
+     * Marker interface for Docker Compose availability
+     */
+    private interface DockerComposeAvailability {
+        /**
+         * Detects Docker Compose V1/V2 availability
+         */
+        private static Optional<DockerComposeAvailability> detect(ExecOperations execOperations, String dockerPath) {
+            Optional<String> composePath = getDockerComposePath();
+            if (composePath.isPresent()) {
+                if (runCommand(execOperations, composePath.get(), "version").isSuccess()) {
+                    return Optional.of(new DockerComposeV1Availability());
+                }
+            }
+
+            if (runCommand(execOperations, dockerPath, "compose", "version").isSuccess()) {
+                return Optional.of(new DockerComposeV2Availability());
+            }
+
+            return Optional.empty();
+        }
+
+        /**
+         * Searches the entries in {@link #DOCKER_COMPOSE_BINARIES} for the Docker Compose CLI. This method does
+         * not check whether the installation appears usable, see {@link #getDockerAvailability()} instead.
+         *
+         * @return the path to a CLI, if available.
+         */
+        private static Optional<String> getDockerComposePath() {
+            // Check if the Docker binary exists
+            return Arrays.asList(DOCKER_COMPOSE_BINARIES).stream().filter(path -> new File(path).exists()).findFirst();
+        }
+
     }
 
+    /**
+     * Docker Compose V1 availability
+     */
+    public static class DockerComposeV1Availability implements DockerComposeAvailability {}
+
+    /**
+     * Docker Compose V2 availability
+     */
+    public static class DockerComposeV2Availability implements DockerComposeAvailability {}
+
     /**
      * This class models the result of running a command. It captures the exit code, standard output and standard error.
      */