[9.3] ESQL: Fix incorrectly optimized fork with nullify unmapped_fields (#143030)

.backportrc.json

@@ -1,9 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "9.2", "9.1", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "9.3", "9.2", "9.1", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v9.3.0$" : "main",
+    "^v9.4.0$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/hooks/pre-command

@@ -145,19 +145,80 @@ EOF
 EOF
 fi
 
+# =============================================================================
+# PART 1: Test seed retrieval for ANY retry
+# =============================================================================
+# When a job is retried (regardless of SMART_RETRIES setting), retrieve and use
+# the same test seed from the original job to ensure reproducible test failures.
+# =============================================================================
+
+# Initialize variables that may be reused by smart retry logic below
+BUILD_JSON=""
+ORIGIN_JOB_ID=""
+BUILD_SCAN_ID=""
+BUILD_SCAN_URL=""
+TESTS_SEED=""
+
+if [[ "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]]; then
+  echo "--- Retrieving test seed from original job"
+
+  if BUILD_JSON=$(curl --max-time 30 -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}?include_retried_jobs=true" 2>/dev/null); then
+    if ORIGIN_JOB_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg jobId "$BUILDKITE_JOB_ID" ' .jobs[] | select(.id == $jobId) | .retry_source.job_id' 2>/dev/null) && [ "$ORIGIN_JOB_ID" != "null" ] && [ -n "$ORIGIN_JOB_ID" ]; then
+
+      # Retrieve test seed from Buildkite metadata
+      TESTS_SEED=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["tests-seed-" + $job_id]' 2>/dev/null)
+
+      # Retrieve build scan URL and ID for smart retry (reused in Part 2)
+      BUILD_SCAN_URL=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-" + $job_id]' 2>/dev/null)
+      BUILD_SCAN_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-id-" + $job_id]' 2>/dev/null)
+
+      if [[ -n "$TESTS_SEED" ]] && [[ "$TESTS_SEED" != "null" ]]; then
+        echo "Using test seed $TESTS_SEED from original job $ORIGIN_JOB_ID"
+        export TESTS_SEED
+      else
+        echo "Warning: Could not find test seed in metadata for original job."
+        echo "Tests will use a new random seed."
+        TESTS_SEED=""
+      fi
+    else
+      echo "Warning: Could not find origin job ID for retry."
+      echo "Tests will use a new random seed."
+    fi
+  else
+    echo "Warning: Failed to fetch build information from Buildkite API"
+    echo "Tests will use a new random seed."
+  fi
+fi
+
+# =============================================================================
+# PART 2: Smart retry test filtering (only when SMART_RETRIES=true)
+# =============================================================================
+# When smart retries are enabled, fetch the list of failed tests from the
+# original job and filter this retry to only run those tests.
+# =============================================================================
 if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]]; then
   echo "--- Resolving previously failed tests"
   SMART_RETRY_STATUS="disabled"
   SMART_RETRY_DETAILS=""
 
-  if BUILD_JSON=$(curl --max-time 30 -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}?include_retried_jobs=true" 2>/dev/null); then
-    if ORIGIN_JOB_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg jobId "$BUILDKITE_JOB_ID" ' .jobs[] | select(.id == $jobId) | .retry_source.job_id' 2>/dev/null) && [ "$ORIGIN_JOB_ID" != "null" ] && [ -n "$ORIGIN_JOB_ID" ]; then
-      
-      # Attempt to retrieve build scan ID directly from metadata
-      BUILD_SCAN_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-id-" + $job_id]' 2>/dev/null)
+  # Check if we already have the build info from seed retrieval above
+  if [[ -z "$BUILD_JSON" ]]; then
+    # Fetch build info if not already available (shouldn't happen, but be safe)
+    BUILD_JSON=$(curl --max-time 30 -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}?include_retried_jobs=true" 2>/dev/null) || BUILD_JSON=""
+  fi
 
-      # Retrieve build scan URL for annotation
-      BUILD_SCAN_URL=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-" + $job_id]' 2>/dev/null)
+  if [[ -n "$BUILD_JSON" ]]; then
+    # Get origin job ID if not already available
+    if [[ -z "$ORIGIN_JOB_ID" ]] || [[ "$ORIGIN_JOB_ID" == "null" ]]; then
+      ORIGIN_JOB_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg jobId "$BUILDKITE_JOB_ID" ' .jobs[] | select(.id == $jobId) | .retry_source.job_id' 2>/dev/null) || ORIGIN_JOB_ID=""
+    fi
+
+    if [[ -n "$ORIGIN_JOB_ID" ]] && [[ "$ORIGIN_JOB_ID" != "null" ]]; then
+      # Get build scan ID if not already available
+      if [[ -z "$BUILD_SCAN_ID" ]] || [[ "$BUILD_SCAN_ID" == "null" ]]; then
+        BUILD_SCAN_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-id-" + $job_id]' 2>/dev/null) || BUILD_SCAN_ID=""
+        BUILD_SCAN_URL=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-" + $job_id]' 2>/dev/null) || BUILD_SCAN_URL=""
+      fi
 
       if [[ -n "$BUILD_SCAN_ID" ]] && [[ "$BUILD_SCAN_ID" != "null" ]]; then
         # Validate BUILD_SCAN_ID format to prevent injection attacks
@@ -180,7 +241,7 @@ if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]];
             --max-time 30 \
             --header 'accept: application/json' \
             --header "authorization: Bearer $DEVELOCITY_API_ACCESS_KEY" \
-            --header 'content-type: application/json' 2>/dev/null | jq '.' &> .failed-test-history.json; then
+            --header 'content-type: application/json' 2>/dev/null | jq --arg testseed "${TESTS_SEED:-}" '. + {testseed: $testseed}' &> .failed-test-history.json; then
 
             # Set secure file permissions
             chmod 600 .failed-test-history.json
@@ -196,7 +257,7 @@ if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]];
               ORIGIN_JOB_NAME="previous attempt"
             fi
 
-            echo "✓ Smart retry enabled: filtering to $FILTERED_WORK_UNITS work units"
+            echo "Smart retry enabled: filtering to $FILTERED_WORK_UNITS work units"
 
             # Create Buildkite annotation for visibility
             # Use unique context per job to support multiple retries

.buildkite/hooks/pre-command.bat

@@ -28,28 +28,95 @@ for /f "delims=" %%i in ('vault read -field^=token secret/ci/elastic-elasticsear
 
 bash.exe -c "nohup bash .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &"
 
-REM Smart retries implementation
+REM =============================================================================
+REM PART 1: Test seed retrieval for ANY retry
+REM =============================================================================
+REM When a job is retried (regardless of SMART_RETRIES setting), retrieve and use
+REM the same test seed from the original job to ensure reproducible test failures.
+REM =============================================================================
+set ORIGIN_JOB_ID=
+set BUILD_SCAN_ID=
+set BUILD_SCAN_URL=
+set TESTS_SEED=
+
+if defined BUILDKITE_RETRY_COUNT (
+  if %BUILDKITE_RETRY_COUNT% GTR 0 (
+    echo --- Retrieving test seed from original job
+
+    REM Fetch build information from Buildkite API
+    curl --max-time 30 -H "Authorization: Bearer %BUILDKITE_API_TOKEN%" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/%BUILDKITE_PIPELINE_SLUG%/builds/%BUILDKITE_BUILD_NUMBER%?include_retried_jobs=true" -o .build-info.json 2>nul
+
+    if exist .build-info.json (
+      REM Extract origin job ID
+      for /f "delims=" %%i in ('jq -r --arg jobId "%BUILDKITE_JOB_ID%" ".jobs[] | select(.id == $jobId) | .retry_source.job_id" .build-info.json 2^>nul') do set ORIGIN_JOB_ID=%%i
+
+      if defined ORIGIN_JOB_ID (
+        if not "!ORIGIN_JOB_ID!"=="null" (
+          REM Retrieve test seed directly from Buildkite metadata
+          for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"tests-seed-\" + $job_id]" .build-info.json 2^>nul') do set TESTS_SEED=%%i
+
+          REM Retrieve build scan URL and ID for smart retry (reused in Part 2)
+          for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_URL=%%i
+          for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-id-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_ID=%%i
+
+          if defined TESTS_SEED (
+            if not "!TESTS_SEED!"=="null" (
+              echo Using test seed !TESTS_SEED! from original job !ORIGIN_JOB_ID!
+            ) else (
+              echo Warning: Could not find test seed in metadata for original job.
+              echo Tests will use a new random seed.
+              set TESTS_SEED=
+            )
+          ) else (
+            echo Warning: Could not find test seed in metadata for original job.
+            echo Tests will use a new random seed.
+          )
+        ) else (
+          echo Warning: Could not find origin job ID for retry.
+          echo Tests will use a new random seed.
+        )
+      ) else (
+        echo Warning: Could not find origin job ID for retry.
+        echo Tests will use a new random seed.
+      )
+    ) else (
+      echo Warning: Failed to fetch build information from Buildkite API
+      echo Tests will use a new random seed.
+    )
+  )
+)
+
+REM =============================================================================
+REM PART 2: Smart retry test filtering (only when SMART_RETRIES=true)
+REM =============================================================================
+REM When smart retries are enabled, fetch the list of failed tests from the
+REM original job and filter this retry to only run those tests.
+REM =============================================================================
 if "%SMART_RETRIES%"=="true" (
   if defined BUILDKITE_RETRY_COUNT (
     if %BUILDKITE_RETRY_COUNT% GTR 0 (
       echo --- Resolving previously failed tests
       set SMART_RETRY_STATUS=disabled
       set SMART_RETRY_DETAILS=
 
-      REM Fetch build information from Buildkite API
-      curl --max-time 30 -H "Authorization: Bearer %BUILDKITE_API_TOKEN%" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/%BUILDKITE_PIPELINE_SLUG%/builds/%BUILDKITE_BUILD_NUMBER%?include_retried_jobs=true" -o .build-info.json 2>nul
+      REM Check if we need to fetch build info (should already exist from Part 1)
+      if not exist .build-info.json (
+        curl --max-time 30 -H "Authorization: Bearer %BUILDKITE_API_TOKEN%" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/%BUILDKITE_PIPELINE_SLUG%/builds/%BUILDKITE_BUILD_NUMBER%?include_retried_jobs=true" -o .build-info.json 2>nul
+      )
 
       if exist .build-info.json (
-        REM Extract origin job ID
-        for /f "delims=" %%i in ('jq -r --arg jobId "%BUILDKITE_JOB_ID%" ".jobs[] | select(.id == $jobId) | .retry_source.job_id" .build-info.json 2^>nul') do set ORIGIN_JOB_ID=%%i
+        REM Get origin job ID if not already set
+        if not defined ORIGIN_JOB_ID (
+          for /f "delims=" %%i in ('jq -r --arg jobId "%BUILDKITE_JOB_ID%" ".jobs[] | select(.id == $jobId) | .retry_source.job_id" .build-info.json 2^>nul') do set ORIGIN_JOB_ID=%%i
+        )
 
         if defined ORIGIN_JOB_ID (
           if not "!ORIGIN_JOB_ID!"=="null" (
-            REM Extract build scan ID directly (new way)
-            for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-id-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_ID=%%i
-
-            REM Retrieve build scan URL for annotation
-            for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_URL=%%i
+            REM Get build scan ID if not already set
+            if not defined BUILD_SCAN_ID (
+              for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-id-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_ID=%%i
+              for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_URL=%%i
+            )
 
             if defined BUILD_SCAN_ID (
               if not "!BUILD_SCAN_ID!"=="null" (
@@ -72,7 +139,7 @@ if "%SMART_RETRIES%"=="true" (
                   timeout /t !delay! /nobreak >nul 2>&1
 
                   REM Fetch failed tests from Develocity API (curl will auto-decompress gzip with --compressed)
-                  curl --compressed --request GET --url "!DEVELOCITY_FAILED_TEST_API_URL!" --max-filesize 10485760 --max-time 30 --header "accept: application/json" --header "authorization: Bearer %DEVELOCITY_API_ACCESS_KEY%" --header "content-type: application/json" 2>nul | jq "." > .failed-test-history.json 2>nul
+                  curl --compressed --request GET --url "!DEVELOCITY_FAILED_TEST_API_URL!" --max-filesize 10485760 --max-time 30 --header "accept: application/json" --header "authorization: Bearer %DEVELOCITY_API_ACCESS_KEY%" --header "content-type: application/json" 2>nul | jq --arg testseed "!TESTS_SEED!" ". + {testseed: $testseed}" > .failed-test-history.json 2>nul
 
                   if exist .failed-test-history.json (
                     REM Set restrictive file permissions (owner only)
@@ -90,7 +157,7 @@ if "%SMART_RETRIES%"=="true" (
                     if not defined ORIGIN_JOB_NAME set ORIGIN_JOB_NAME=previous attempt
                     if "!ORIGIN_JOB_NAME!"=="null" set ORIGIN_JOB_NAME=previous attempt
 
-                    echo ✓ Smart retry enabled: filtering to !FILTERED_WORK_UNITS! work units
+                    echo Smart retry enabled: filtering to !FILTERED_WORK_UNITS! work units
 
                     REM Create Buildkite annotation for visibility
                     echo Rerunning failed build job [!ORIGIN_JOB_NAME!]^(!BUILD_SCAN_URL!^) > .smart-retry-annotation.txt
@@ -173,10 +240,11 @@ set "_DEVELOCITY_API_ACCESS_KEY=%DEVELOCITY_API_ACCESS_KEY%"
 set "_BUILDKITE_API_TOKEN=%BUILDKITE_API_TOKEN%"
 set "_JAVA_HOME=%JAVA_HOME%"
 set "_JAVA16_HOME=%JAVA16_HOME%"
+set "_TESTS_SEED=%TESTS_SEED%"
 
 REM End local scope and restore critical variables to parent environment
 REM This ensures bash scripts can access WORKSPACE, GRADLEW, and other variables
-ENDLOCAL && set "WORKSPACE=%_WORKSPACE%" && set "GRADLEW=%_GRADLEW%" && set "GRADLEW_BAT=%_GRADLEW_BAT%" && set "BUILD_NUMBER=%_BUILD_NUMBER%" && set "JOB_BRANCH=%_JOB_BRANCH%" && set "GH_TOKEN=%_GH_TOKEN%" && set "GRADLE_BUILD_CACHE_USERNAME=%_GRADLE_BUILD_CACHE_USERNAME%" && set "GRADLE_BUILD_CACHE_PASSWORD=%_GRADLE_BUILD_CACHE_PASSWORD%" && set "DEVELOCITY_ACCESS_KEY=%_DEVELOCITY_ACCESS_KEY%" && set "DEVELOCITY_API_ACCESS_KEY=%_DEVELOCITY_API_ACCESS_KEY%" && set "BUILDKITE_API_TOKEN=%_BUILDKITE_API_TOKEN%" && set "JAVA_HOME=%_JAVA_HOME%" && set "JAVA16_HOME=%_JAVA16_HOME%"
+ENDLOCAL && set "WORKSPACE=%_WORKSPACE%" && set "GRADLEW=%_GRADLEW%" && set "GRADLEW_BAT=%_GRADLEW_BAT%" && set "BUILD_NUMBER=%_BUILD_NUMBER%" && set "JOB_BRANCH=%_JOB_BRANCH%" && set "GH_TOKEN=%_GH_TOKEN%" && set "GRADLE_BUILD_CACHE_USERNAME=%_GRADLE_BUILD_CACHE_USERNAME%" && set "GRADLE_BUILD_CACHE_PASSWORD=%_GRADLE_BUILD_CACHE_PASSWORD%" && set "DEVELOCITY_ACCESS_KEY=%_DEVELOCITY_ACCESS_KEY%" && set "DEVELOCITY_API_ACCESS_KEY=%_DEVELOCITY_API_ACCESS_KEY%" && set "BUILDKITE_API_TOKEN=%_BUILDKITE_API_TOKEN%" && set "JAVA_HOME=%_JAVA_HOME%" && set "JAVA16_HOME=%_JAVA16_HOME%" && set "TESTS_SEED=%_TESTS_SEED%"
 
 bash.exe -c "bash .buildkite/scripts/get-latest-test-mutes.sh"
 

.buildkite/pipelines/intake.template.yml

@@ -107,7 +107,7 @@ steps:
   - trigger: elasticsearch-dra-workflow
     label: Trigger DRA snapshot workflow
     async: true
-    branches: "main 9.* 8.* 7.17"
+    branches: "main 9.* 8.*"
     build:
       branch: "$BUILDKITE_BRANCH"
       commit: "$BUILDKITE_COMMIT"

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.19.9", "9.1.9", "9.2.3", "9.3.0"]
+            BWC_VERSION: ["8.19.13", "9.2.7", "9.3.2", "9.4.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -108,7 +108,7 @@ steps:
   - trigger: elasticsearch-dra-workflow
     label: Trigger DRA snapshot workflow
     async: true
-    branches: "main 9.* 8.* 7.17"
+    branches: "main 9.* 8.*"
     build:
       branch: "$BUILDKITE_BRANCH"
       commit: "$BUILDKITE_COMMIT"

.buildkite/pipelines/periodic-java-ea.template.yml

@@ -74,7 +74,7 @@ steps:
           machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
-          GRADLE_PARAMS: "-Druntime.java=$$JAVA_EA_VERSION"
+          GRADLE_PARAMS: "-Druntime.java=$JAVA_EA_VERSION"
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
       - label: "{{matrix.BWC_VERSION}} / matrix-bwc"
         command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION  -Dbwc.checkout.align=true v$$BWC_VERSION#bwcTest