Merge branch 'story/state-v2-base' into story/state-v2-changelog

Author: anishshri-db

.github/workflows/benchmark.yml

@@ -105,7 +105,7 @@ jobs:
         run: cd tpcds-kit/tools && make OS=LINUX
       - name: Install Java ${{ github.event.inputs.jdk }}
         if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: zulu
           java-version: ${{ github.event.inputs.jdk }}
@@ -157,7 +157,7 @@ jobs:
         restore-keys: |
           benchmark-coursier-${{ github.event.inputs.jdk }}
     - name: Install Java ${{ github.event.inputs.jdk }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: ${{ github.event.inputs.jdk }}
@@ -177,7 +177,7 @@ jobs:
         # In benchmark, we use local as master so set driver memory only. Note that GitHub Actions has 7 GB memory limit.
         bin/spark-submit \
           --driver-memory 6g --class org.apache.spark.benchmark.Benchmarks \
-          --jars "`find . -name '*-SNAPSHOT-tests.jar' -o -name '*avro*-SNAPSHOT.jar' | paste -sd ',' -`" \
+          --jars "`find . -name '*-SNAPSHOT-tests.jar' -o -name '*avro*-SNAPSHOT.jar' | paste -sd ',' -`,`find ~/.cache/coursier -name 'curator-test-*.jar'`" \
           "`find . -name 'spark-core*-SNAPSHOT-tests.jar'`" \
           "${{ github.event.inputs.class }}"
         # To keep the directory structure and file permissions, tar them

.github/workflows/build_and_test.yml

@@ -98,7 +98,8 @@ jobs:
               \"java-other-versions\": \"true\",
               \"lint\" : \"true\",
               \"k8s-integration-tests\" : \"true\",
-              \"breaking-changes-buf\" : \"true\",
+              \"buf\" : \"true\",
+              \"ui\" : \"true\",
             }"
           echo $precondition # For debugging
           # Remove `\n` to avoid "Invalid format" error
@@ -149,7 +150,7 @@ jobs:
           - >-
             mllib-local, mllib, graphx
           - >-
-            streaming, sql-kafka-0-10, streaming-kafka-0-10,
+            streaming, sql-kafka-0-10, streaming-kafka-0-10, streaming-kinesis-asl,
             yarn, kubernetes, hadoop-cloud, spark-ganglia-lgpl,
             connect, protobuf
         # Here, we split Hive and SQL tests into some of slow ones and the rest of them.
@@ -240,12 +241,12 @@ jobs:
           ./dev/free_disk_space
         fi
     - name: Install Java ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: ${{ matrix.java }}
     - name: Install Python 3.9
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       # We should install one Python that is higher than 3+ for SQL and Yarn because:
       # - SQL component also has Python related tests, for example, IntegratedUDFTestUtils.
       # - Yarn has a Python specific test too, for example, YarnClusterSuite.
@@ -267,6 +268,10 @@ jobs:
         export TERM=vt100
         # Hive "other tests" test needs larger metaspace size based on experiment.
         if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" == "org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi
+        // SPARK-46283: should delete the following env replacement after SPARK 3.x EOL
+        if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "${{ inputs.branch }}" =~ ^branch-3 ]]; then 
+          MODULES_TO_TEST=${MODULES_TO_TEST//streaming-kinesis-asl, /}
+        fi
         export SERIAL_SBT_TESTS=1
         ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS"
     - name: Upload test results to report
@@ -427,7 +432,7 @@ jobs:
           ./dev/free_disk_space_container
         fi
     - name: Install Java ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: ${{ matrix.java }}
@@ -537,7 +542,7 @@ jobs:
           ./dev/free_disk_space_container
         fi
     - name: Install Java ${{ inputs.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: ${{ inputs.java }}
@@ -556,11 +561,10 @@ jobs:
         name: test-results-sparkr--${{ inputs.java }}-${{ inputs.hadoop }}-hive2.3
         path: "**/target/test-reports/*.xml"
 
-  breaking-changes-buf:
+  buf:
     needs: [precondition]
-    if: (!cancelled()) && fromJson(needs.precondition.outputs.required).breaking-changes-buf == 'true'
-    # Change 'branch-3.5' to 'branch-4.0' in master branch after cutting branch-4.0 branch.
-    name: Breaking change detection with Buf (branch-3.5)
+    if: (!cancelled()) && fromJson(needs.precondition.outputs.required).buf == 'true'
+    name: Protobuf breaking change detection and Python CodeGen check
     runs-on: ubuntu-22.04
     steps:
     - name: Checkout Spark repository
@@ -579,12 +583,26 @@ jobs:
       uses: bufbuild/buf-setup-action@v1
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
-    - name: Detect breaking changes
+    - name: Protocol Buffers Linter
+      uses: bufbuild/buf-lint-action@v1
+      with:
+        input: core/src/main/protobuf
+    # Change 'branch-3.5' to 'branch-4.0' in master branch after cutting branch-4.0 branch.
+    - name: Breaking change detection against branch-3.5
       uses: bufbuild/buf-breaking-action@v1
       with:
         input: connector/connect/common/src/main
         against: 'https://github.com/apache/spark.git#branch=branch-3.5,subdir=connector/connect/common/src/main'
-
+    - name: Install Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+    - name: Install dependencies for Python CodeGen check
+      run: |
+        python3.9 -m pip install 'black==23.9.1' 'protobuf==4.25.1' 'mypy==0.982' 'mypy-protobuf==3.3.0'
+        python3.9 -m pip list
+    - name: Python CodeGen check
+      run: ./dev/connect-check-protos.py
 
   # Static analysis, and documentation build
   lint:
@@ -651,7 +669,7 @@ jobs:
           ./dev/free_disk_space_container
         fi
     - name: Install Java ${{ inputs.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: ${{ inputs.java }}
@@ -696,17 +714,19 @@ jobs:
         python3.9 -m pip install 'pandas-stubs==1.2.0.53' ipython 'grpcio==1.59.3' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0'
     - name: Python linter
       run: PYTHON_EXECUTABLE=python3.9 ./dev/lint-python
-    - name: Install dependencies for Python code generation check
-      if: inputs.branch != 'branch-3.3' && inputs.branch != 'branch-3.4'
+    # Should delete this section after SPARK 3.5 EOL.
+    - name: Install dependencies for Python code generation check for branch-3.5
+      if: inputs.branch == 'branch-3.5'
       run: |
         # See more in "Installation" https://docs.buf.build/installation#tarball
         curl -LO https://github.com/bufbuild/buf/releases/download/v1.28.1/buf-Linux-x86_64.tar.gz
         mkdir -p $HOME/buf
         tar -xvzf buf-Linux-x86_64.tar.gz -C $HOME/buf --strip-components 1
         rm buf-Linux-x86_64.tar.gz
         python3.9 -m pip install 'protobuf==4.25.1' 'mypy-protobuf==3.3.0'
-    - name: Python code generation check
-      if: inputs.branch != 'branch-3.3' && inputs.branch != 'branch-3.4'
+    # Should delete this section after SPARK 3.5 EOL.
+    - name: Python code generation check for branch-3.5
+      if: inputs.branch == 'branch-3.5'
       run: if test -f ./dev/connect-check-protos.py; then PATH=$PATH:$HOME/buf/bin PYTHON_EXECUTABLE=python3.9 ./dev/connect-check-protos.py; fi
     # Should delete this section after SPARK 3.5 EOL.
     - name: Install JavaScript linter dependencies for branch-3.3, branch-3.4, branch-3.5
@@ -741,9 +761,7 @@ jobs:
         Rscript -e "devtools::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')"
     - name: Install dependencies for documentation generation
       run: |
-        # Pin the MarkupSafe to 2.0.1 to resolve the CI error.
-        #   See also https://issues.apache.org/jira/browse/SPARK-38279.
-        python3.9 -m pip install 'sphinx==4.2.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 'markupsafe==2.0.1' 'pyzmq<24.0.0'
+        python3.9 -m pip install 'sphinx==4.2.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0'
         python3.9 -m pip install ipython_genutils # See SPARK-38517
         python3.9 -m pip install sphinx_plotly_directive 'numpy>=1.20.0' pyarrow pandas 'plotly>=4.8'
         python3.9 -m pip install 'docutils<0.18.0' # See SPARK-39421
@@ -817,7 +835,7 @@ jobs:
         restore-keys: |
           java${{ matrix.java }}-maven-
     - name: Install Java ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: ${{ matrix.java }}
@@ -872,7 +890,7 @@ jobs:
         restore-keys: |
           tpcds-coursier-
     - name: Install Java ${{ inputs.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: ${{ inputs.java }}
@@ -979,7 +997,7 @@ jobs:
         restore-keys: |
           docker-integration-coursier-
     - name: Install Java ${{ inputs.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: zulu
         java-version: ${{ inputs.java }}
@@ -1039,7 +1057,7 @@ jobs:
           restore-keys: |
             k8s-integration-coursier-
       - name: Install Java ${{ inputs.java }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: zulu
           java-version: ${{ inputs.java }}
@@ -1074,3 +1092,22 @@ jobs:
         with:
           name: spark-on-kubernetes-it-log
           path: "**/target/integration-tests.log"
+
+  ui:
+    needs: [precondition]
+    if: fromJson(needs.precondition.outputs.required).ui == 'true'
+    name: Run Spark UI tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 300
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+          cache-dependency-path: ui-test/package-lock.json
+      - run: |
+          cd ui-test
+          npm install --save-dev
+          node --experimental-vm-modules node_modules/.bin/jest

.github/workflows/build_maven_java21.yml

@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Build using Maven (master, Scala 2.13, Hadoop 3, JDK 21)"
+
+on:
+  schedule:
+    - cron: '0 14 * * *'
+
+jobs:
+  run-build:
+    permissions:
+      packages: write
+    name: Run
+    uses: ./.github/workflows/maven_test.yml
+    if: github.repository == 'apache/spark'
+    with:
+      java: 21

.github/workflows/build_python.yml

@@ -25,6 +25,9 @@ on:
 
 jobs:
   run-build:
+    strategy:
+      matrix:
+        pyversion: ["pypy3,python3.10", "python3.11,python3.12"]
     permissions:
       packages: write
     name: Run
@@ -36,7 +39,7 @@ jobs:
       hadoop: hadoop3
       envs: >-
         {
-          "PYTHON_TO_TEST": "pypy3,python3.10,python3.11,python3.12"
+          "PYTHON_TO_TEST": "${{ matrix.pyversion }}"
         }
       jobs: >-
         {

.github/workflows/maven_test.yml

@@ -150,12 +150,12 @@ jobs:
           restore-keys: |
             java${{ matrix.java }}-maven-
       - name: Install Java ${{ matrix.java }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: zulu
           java-version: ${{ matrix.java }}
       - name: Install Python 3.9
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         # We should install one Python that is higher than 3+ for SQL and Yarn because:
         # - SQL component also has Python related tests, for example, IntegratedUDFTestUtils.
         # - Yarn has a Python specific test too, for example, YarnClusterSuite.

.github/workflows/notify_test_workflow.yml

@@ -36,7 +36,7 @@ jobs:
       checks: write
     steps:
       - name: "Notify test workflow"
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |

.github/workflows/publish_snapshot.yml

@@ -53,13 +53,13 @@ jobs:
           snapshot-maven-
     - name: Install Java 8 for branch-3.x
       if: matrix.branch == 'branch-3.5' || matrix.branch == 'branch-3.4' || matrix.branch == 'branch-3.3'
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: temurin
         java-version: 8
     - name: Install Java 17
       if: matrix.branch != 'branch-3.5' && matrix.branch != 'branch-3.4' && matrix.branch != 'branch-3.3'
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: temurin
         java-version: 17

.github/workflows/update_build_status.yml

@@ -32,7 +32,7 @@ jobs:
       checks: write
     steps:
       - name: "Update build status"
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |

R/pkg/tests/fulltests/test_sparkSQL.R

@@ -4103,10 +4103,7 @@ test_that("catalog APIs, listCatalogs, setCurrentCatalog, currentCatalog", {
   expect_equal(currentCatalog(), "spark_catalog")
   expect_error(setCurrentCatalog("spark_catalog"), NA)
   expect_error(setCurrentCatalog("zxwtyswklpf"),
-               paste0("Error in setCurrentCatalog : ",
-               "org.apache.spark.sql.connector.catalog.CatalogNotFoundException: ",
-               "Catalog 'zxwtyswklpf' plugin class not found: ",
-               "spark.sql.catalog.zxwtyswklpf is not defined"))
+               "[CATALOG_NOT_FOUND]*`zxwtyswklpf`*")
   catalogs <- collect(listCatalogs())
 })