Merge branch 'main' into dependabot/gradle/plugins/repository-gcs/com.google.api-gax-2.62.0

Signed-off-by: Daniel Widdis <[email protected]>

Author: dbwiddis

.ci/bwcVersions

@@ -44,4 +44,5 @@ BWC_VERSION:
   - "2.18.1"
   - "2.19.0"
   - "2.19.1"
+  - "2.19.2"
   - "2.20.0"

.github/CODEOWNERS

@@ -11,7 +11,7 @@
 #   3. Use the command palette to run the CODEOWNERS: Show owners of current file command, which will display all code owners for the current file.
 
 # Default ownership for all repo files
-* @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dblock @dbwiddis @gbbafna @jainankitk @kotwanikunal @linuxpi @mch2 @msfroh @nknize @owaiskazi19  @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+* @anasalkouz @andrross @ashking94 @bugmakerrrrrr @Bukhtawar @CEHENKLE @cwperks @dblock @dbwiddis @gbbafna @jainankitk @kotwanikunal @linuxpi @mch2 @msfroh @nknize @owaiskazi19  @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 
 /modules/lang-painless/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 /modules/parent-join/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah

.github/benchmark-configs.json

@@ -2,7 +2,7 @@
   "name": "Cluster and opensearch-benchmark configurations",
   "id_1": {
     "description": "Indexing only configuration for NYC_TAXIS workload",
-    "supported_major_versions": ["2", "3"],
+    "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
@@ -19,7 +19,7 @@
   },
   "id_2": {
     "description": "Indexing only configuration for HTTP_LOGS workload",
-    "supported_major_versions": ["2", "3"],
+    "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
@@ -41,7 +41,7 @@
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "nyc_taxis",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"nyc_taxis_1_shard\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"nyc_taxis_1_shard\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -52,81 +52,13 @@
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
   "id_4": {
-    "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-3.0.0",
-    "supported_major_versions": ["3"],
-    "cluster-benchmark-configs": {
-      "SINGLE_NODE_CLUSTER": "true",
-      "MIN_DISTRIBUTION": "true",
-      "TEST_WORKLOAD": "http_logs",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"http_logs_1_shard\"}",
-      "CAPTURE_NODE_STAT": "true",
-      "TEST_PROCEDURE": "restore-from-snapshot"
-    },
-    "cluster_configuration": {
-      "size": "Single-Node",
-      "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
-  },
-  "id_5": {
     "description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-3.0.0",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}",
-      "CAPTURE_NODE_STAT": "true",
-      "TEST_PROCEDURE": "restore-from-snapshot"
-    },
-    "cluster_configuration": {
-      "size": "Single-Node",
-      "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
-  },
-  "id_6": {
-    "description": "Search only test-procedure for NYC_TAXIS, uses snapshot to restore the data for OS-2.x",
-    "supported_major_versions": ["2"],
-    "cluster-benchmark-configs": {
-      "SINGLE_NODE_CLUSTER": "true",
-      "MIN_DISTRIBUTION": "true",
-      "TEST_WORKLOAD": "nyc_taxis",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"nyc_taxis_1_shard\"}",
-      "CAPTURE_NODE_STAT": "true",
-      "TEST_PROCEDURE": "restore-from-snapshot"
-    },
-    "cluster_configuration": {
-      "size": "Single-Node",
-      "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
-  },
-  "id_7": {
-    "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-2.x",
-    "supported_major_versions": ["2"],
-    "cluster-benchmark-configs": {
-      "SINGLE_NODE_CLUSTER": "true",
-      "MIN_DISTRIBUTION": "true",
-      "TEST_WORKLOAD": "http_logs",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"http_logs_1_shard\"}",
-      "CAPTURE_NODE_STAT": "true",
-      "TEST_PROCEDURE": "restore-from-snapshot"
-    },
-    "cluster_configuration": {
-      "size": "Single-Node",
-      "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
-  },
-  "id_8": {
-    "description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-2.x",
-    "supported_major_versions": ["2"],
-    "cluster-benchmark-configs": {
-      "SINGLE_NODE_CLUSTER": "true",
-      "MIN_DISTRIBUTION": "true",
-      "TEST_WORKLOAD": "big5",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"big5_1_shard_ordered\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -136,9 +68,9 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
-  "id_9": {
+  "id_5": {
     "description": "Indexing and search configuration for pmc workload",
-    "supported_major_versions": ["2", "3"],
+    "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
@@ -152,9 +84,9 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   },
-  "id_10": {
+  "id_6": {
     "description": "Indexing only configuration for stack-overflow workload",
-    "supported_major_versions": ["2", "3"],
+    "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
@@ -168,15 +100,15 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   },
-  "id_11": {
+  "id_7": {
     "description": "Search only test-procedure for big5 with concurrent segment search setting enabled",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
       "ADDITIONAL_CONFIG": "search.concurrent_segment_search.enabled:true",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -186,15 +118,15 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
-  "id_12": {
+  "id_8": {
     "description": "Search only test-procedure for big5 with concurrent segment search mode as all",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
       "ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:all",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -204,15 +136,15 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
-  "id_13": {
+  "id_9": {
     "description": "Search only test-procedure for big5 with concurrent segment search mode as auto",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
       "ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:auto",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -222,31 +154,14 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
-  "id_14": {
+  "id_10": {
     "description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-3.0.0. Enables range query approximation.",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
       "ADDITIONAL_CONFIG": "opensearch.experimental.feature.approximate_point_range_query.enabled:true",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}",
-      "CAPTURE_NODE_STAT": "true",
-      "TEST_PROCEDURE": "restore-from-snapshot"
-    },
-    "cluster_configuration": {
-      "size": "Single-Node",
-      "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
-  },
-  "id_15": {
-    "description": "Search only test-procedure for big5, uses lucene-10 index snapshot to restore the data for OS-3.0.0",
-    "supported_major_versions": ["3"],
-    "cluster-benchmark-configs": {
-      "SINGLE_NODE_CLUSTER": "true",
-      "MIN_DISTRIBUTION": "true",
-      "TEST_WORKLOAD": "big5",
       "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
@@ -257,9 +172,9 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
-  "id_16": {
+  "id_11": {
     "description": "Benchmarking config for NESTED workload, benchmarks nested queries with inner-hits",
-    "supported_major_versions": ["2", "3"],
+    "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
@@ -273,4 +188,4 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   }
-  }
+}

.github/workflows/benchmark-pull-request.yml

@@ -22,7 +22,7 @@ jobs:
           echo "PR_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
           echo "REPOSITORY=${{ github.event.repository.full_name }}" >> $GITHUB_ENV
           OPENSEARCH_VERSION=$(awk -F '=' '/^opensearch[[:space:]]*=/ {gsub(/[[:space:]]/, "", $2); print $2}' buildSrc/version.properties)
-          echo "OPENSEARCH_VERSION=$OPENSEARCH_VERSION" >> $GITHUB_ENV
+          echo "OPENSEARCH_VERSION=$OPENSEARCH_VERSION-alpha1" >> $GITHUB_ENV
           major_version=$(echo $OPENSEARCH_VERSION | cut -d'.' -f1)
           echo "OPENSEARCH_MAJOR_VERSION=$major_version" >> $GITHUB_ENV
           echo "USER_TAGS=pull_request_number:${{ github.event.issue.number }},repository:OpenSearch" >> $GITHUB_ENV
@@ -147,7 +147,7 @@ jobs:
           distribution: 'temurin'
       - name: Build and Assemble OpenSearch from PR
         run: |
-          ./gradlew :distribution:archives:linux-tar:assemble -Dbuild.snapshot=false
+          ./gradlew :distribution:archives:linux-tar:assemble -Dbuild.snapshot=false -Dbuild.version_qualifier=alpha1
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:

.github/workflows/precommit.yml

@@ -9,6 +9,10 @@ jobs:
       matrix:
         java: [ 21, 23 ]
         os: [ubuntu-latest, windows-latest, macos-latest, macos-13, ubuntu-24.04-arm]
+        include:
+          - java: 21
+            os: 'windows-2025'
+            experimental: true
     steps:
       - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}
@@ -18,6 +22,7 @@ jobs:
           distribution: temurin
           cache: gradle
       - name: Run Gradle (precommit)
+        continue-on-error: ${{ matrix.experimental }}
         shell: bash
         run: |
           ./gradlew javadoc precommit --parallel

CHANGELOG-3.0.md

@@ -21,11 +21,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add execution_hint to cardinality aggregator request (#[17312](https://github.com/opensearch-project/OpenSearch/pull/17312))
 - Arrow Flight RPC plugin with Flight server bootstrap logic and client for internode communication ([#16962](https://github.com/opensearch-project/OpenSearch/pull/16962))
 - Added offset management for the pull-based Ingestion ([#17354](https://github.com/opensearch-project/OpenSearch/pull/17354))
+- Added integ tests for systemd configs ([#17410](https://github.com/opensearch-project/OpenSearch/pull/17410))
+- Add filter function for AbstractQueryBuilder, BoolQueryBuilder, ConstantScoreQueryBuilder([#17409](https://github.com/opensearch-project/OpenSearch/pull/17409))
+- [Star Tree] [Search] Resolving keyword & numeric bucket aggregation with metric aggregation using star-tree ([#17165](https://github.com/opensearch-project/OpenSearch/pull/17165))
+- Added error handling support for the pull-based ingestion ([#17427](https://github.com/opensearch-project/OpenSearch/pull/17427))
+
 
 ### Dependencies
 - Update Apache Lucene to 10.1.0 ([#16366](https://github.com/opensearch-project/OpenSearch/pull/16366))
 - Bump Apache HttpCore5/HttpClient5 dependencies from 5.2.5/5.3.1 to 5.3.1/5.4.1 to support ExtendedSocketOption in HttpAsyncClient ([#16757](https://github.com/opensearch-project/OpenSearch/pull/16757))
 - Bumps `jetty` version from 9.4.55.v20240627 to 9.4.57.v20241219
+- Switch main/3.x to use JDK21 LTS version ([#17515](https://github.com/opensearch-project/OpenSearch/pull/17515))
 
 ### Changed
 - Changed locale provider from COMPAT to CLDR  ([#14345](https://github.com/opensearch-project/OpenSearch/pull/14345))
@@ -67,6 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remove package org.opensearch.action.support.master ([#4856](https://github.com/opensearch-project/OpenSearch/issues/4856))
 - Remove transport-nio plugin ([#16887](https://github.com/opensearch-project/OpenSearch/issues/16887))
 - Remove deprecated 'gateway' settings used to defer cluster recovery ([#3117](https://github.com/opensearch-project/OpenSearch/issues/3117))
+- Remove FeatureFlags.PLUGGABLE_CACHE as the feature is no longer experimental ([#17344](https://github.com/opensearch-project/OpenSearch/pull/17344))
 
 ### Fixed
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
@@ -75,6 +82,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix swapped field formats in nodes API where `total_indexing_buffer_in_bytes` and `total_indexing_buffer` values were reversed ([#17070](https://github.com/opensearch-project/OpenSearch/pull/17070))
 - Add HTTP/2 protocol support to HttpRequest.HttpVersion ([#17248](https://github.com/opensearch-project/OpenSearch/pull/17248))
 - Fix missing bucket in terms aggregation with missing value ([#17418](https://github.com/opensearch-project/OpenSearch/pull/17418))
+- Fix explain action on query rewrite ([#17286](https://github.com/opensearch-project/OpenSearch/pull/17286))
 
 ### Security