[pull] master from apache:master

.github/workflows/benchmark.yml

@@ -66,8 +66,8 @@ jobs:
 
   # Any TPC-DS related updates on this job need to be applied to tpcds-1g job of build_and_test.yml as well
   tpcds-1g-gen:
-    name: "Generate an input dataset for TPCDSQueryBenchmark with SF=1"
-    if: contains(inputs.class, 'TPCDSQueryBenchmark') || contains(inputs.class, '*')
+    name: "Generate an TPC-DS dataset with SF=1"
+    if: contains(inputs.class, 'TPCDSQueryBenchmark') || contains(inputs.class, 'ZStandardTPCDSDataBenchmark') || contains(inputs.class, '*')
     runs-on: ubuntu-latest
     env:
       SPARK_LOCAL_IP: localhost
@@ -98,7 +98,9 @@ jobs:
         id: cache-tpcds-sf-1
         uses: actions/cache@v4
         with:
-          path: ./tpcds-sf-1
+          path: |
+            ./tpcds-sf-1
+            ./tpcds-sf-1-text
           key: tpcds-${{ hashFiles('.github/workflows/benchmark.yml', 'sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala') }}
       - name: Checkout tpcds-kit repository
         if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
@@ -118,7 +120,9 @@ jobs:
           java-version: ${{ inputs.jdk }}
       - name: Generate TPC-DS (SF=1) table data
         if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
-        run: build/sbt "sql/Test/runMain org.apache.spark.sql.GenTPCDSData --dsdgenDir `pwd`/tpcds-kit/tools --location `pwd`/tpcds-sf-1 --scaleFactor 1 --numPartitions 1 --overwrite"
+        run: |
+          build/sbt "sql/Test/runMain org.apache.spark.sql.GenTPCDSData --dsdgenDir `pwd`/tpcds-kit/tools --location `pwd`/tpcds-sf-1 --scaleFactor 1 --numPartitions 1 --overwrite"
+          mkdir -p `pwd`/tpcds-sf-1-text && rm -f `pwd`/tpcds-sf-1-text/* && `pwd`/tpcds-kit/tools/dsdgen -DISTRIBUTIONS `pwd`/tpcds-kit/tools/tpcds.idx -SCALE 1 -DIR `pwd`/tpcds-sf-1-text
 
   benchmark:
     name: "Run benchmarks: ${{ inputs.class }} (JDK ${{ inputs.jdk }}, Scala ${{ inputs.scala }}, ${{ matrix.split }} out of ${{ inputs.num-splits }} splits)"
@@ -138,6 +142,7 @@ jobs:
       # To prevent spark.test.home not being set. See more detail in SPARK-36007.
       SPARK_HOME: ${{ github.workspace }}
       SPARK_TPCDS_DATA: ${{ github.workspace }}/tpcds-sf-1
+      SPARK_TPCDS_DATA_TEXT: ${{ github.workspace }}/tpcds-sf-1-text
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v4
@@ -167,11 +172,13 @@ jobs:
         distribution: zulu
         java-version: ${{ inputs.jdk }}
     - name: Cache TPC-DS generated data
-      if: contains(inputs.class, 'TPCDSQueryBenchmark') || contains(inputs.class, '*')
+      if: contains(inputs.class, 'TPCDSQueryBenchmark') || contains(inputs.class, 'ZStandardTPCDSDataBenchmark') || contains(inputs.class, '*')
       id: cache-tpcds-sf-1
       uses: actions/cache@v4
       with:
-        path: ./tpcds-sf-1
+        path: |
+          ./tpcds-sf-1
+          ./tpcds-sf-1-text
         key: tpcds-${{ hashFiles('.github/workflows/benchmark.yml', 'sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala') }}
     - name: Run benchmarks
       run: |
@@ -188,7 +195,7 @@ jobs:
         # To keep the directory structure and file permissions, tar them
         # See also https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files
         echo "Preparing the benchmark results:"
-        tar -cvf benchmark-results-${{ inputs.jdk }}-${{ inputs.scala }}.tar `git diff --name-only` `git ls-files --others --exclude=tpcds-sf-1 --exclude-standard`
+        tar -cvf benchmark-results-${{ inputs.jdk }}-${{ inputs.scala }}.tar `git diff --name-only` `git ls-files --others --exclude=tpcds-sf-1 --exclude=tpcds-sf-1-text --exclude-standard`
     - name: Upload benchmark results
       uses: actions/upload-artifact@v4
       with:

.github/workflows/build_and_test.yml

@@ -389,6 +389,12 @@ jobs:
       with:
         name: unit-tests-log-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }}
         path: "**/target/*.log"
+    - name: Upload yarn app log files
+      if: ${{ !success() && contains(matrix.modules, 'yarn') }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: yarn-app-log-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }}
+        path: "**/target/test/data/"
 
   infra-image:
     name: "Base image build"

.github/workflows/build_coverage.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / Coverage (master, Scala 2.13, Hadoop 3, JDK 17)"
+name: "Build / Python Coverage (master, Python 3.11)"
 
 on:
   schedule:

.github/workflows/build_java21.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build (master, Scala 2.13, Hadoop 3, JDK 21)"
+name: "Build / Java21 (master, Scala 2.13, Hadoop 3, JDK 21)"
 
 on:
   schedule:

.github/workflows/build_python_3.11_classic_only.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / Python-only Classic-only (master, Python 3.11)"
+name: "Build / Python-only, Classic-only (master, Python 3.11)"
 
 on:
   schedule:

.github/workflows/build_python_3.13_nogil.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / Python-only (master, Python 3.13 no GIL)"
+name: "Build / Python-only (master, Python 3.13, no GIL)"
 
 on:
   schedule:

.github/workflows/build_python_connect.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: Build / Spark Connect Python-only (master, Python 3.11)
+name: Build / Python-only, Connect-only (master, Python 3.11)
 
 on:
   schedule:

.github/workflows/build_python_connect35.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: Build / Spark Connect Python-only (master-server, 35-client, Python 3.11)
+name: Build / Python-only, Connect-only (master-server, branch-3.5-client, Python 3.11)
 
 on:
   schedule:

.github/workflows/build_python_minimum.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / Python-only (master, Python with old dependencies)"
+name: "Build / Python-only (master, Minimum dependencies of PySpark)"
 
 on:
   schedule:

.github/workflows/build_python_ps_minimum.yml

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / Python-only (master, Python PS with old dependencies)"
+name: "Build / Python-only (master, Minimum dependencies of Pandas API on Spark)"
 
 on:
   schedule: