From 21847569437fce6701e182380c390147501f97bd Mon Sep 17 00:00:00 2001 From: panbingkun Date: Mon, 15 Jan 2024 10:22:53 +0800 Subject: [PATCH 1/3] Reapply "[SPARK-46474][INFRA] Upgrade upload-artifact action to v4" This reverts commit c18c87dc70d4dfbb40335fe655f79da21835ff80. --- .github/workflows/benchmark.yml | 2 +- .github/workflows/build_and_test.yml | 26 +++++++++++++------------- .github/workflows/maven_test.yml | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 20ea952ff0a2a..257c80022110c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -185,7 +185,7 @@ jobs: echo "Preparing the benchmark results:" tar -cvf benchmark-results-${{ github.event.inputs.jdk }}-${{ github.event.inputs.scala }}.tar `git diff --name-only` `git ls-files --others --exclude=tpcds-sf-1 --exclude-standard` - name: Upload benchmark results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: benchmark-results-${{ github.event.inputs.jdk }}-${{ github.event.inputs.scala }}-${{ matrix.split }} path: benchmark-results-${{ github.event.inputs.jdk }}-${{ github.event.inputs.scala }}.tar diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 089a9296bcc4e..d29189b444fba 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -272,13 +272,13 @@ jobs: ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS" - name: Upload test results to report if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-results-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }} path: "**/target/test-reports/*.xml" - name: Upload unit tests log files if: ${{ !success() }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit-tests-log-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }} path: "**/target/unit-tests.log" @@ -470,16 +470,16 @@ jobs: - name: Upload test results to report env: ${{ fromJSON(inputs.envs) }} if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: test-results-${{ matrix.modules }}--${{ matrix.java }}-${{ inputs.hadoop }}-hive2.3 + name: test-results-${{ matrix.modules }}--${{ matrix.java }}-${{ inputs.hadoop }}-hive2.3-${{ env.PYTHON_TO_TEST }} path: "**/target/test-reports/*.xml" - name: Upload unit tests log files env: ${{ fromJSON(inputs.envs) }} if: ${{ !success() }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: unit-tests-log-${{ matrix.modules }}--${{ matrix.java }}-${{ inputs.hadoop }}-hive2.3 + name: unit-tests-log-${{ matrix.modules }}--${{ matrix.java }}-${{ inputs.hadoop }}-hive2.3-${{ env.PYTHON_TO_TEST }} path: "**/target/unit-tests.log" sparkr: @@ -556,7 +556,7 @@ jobs: ./dev/run-tests --parallelism 1 --modules sparkr - name: Upload test results to report if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-results-sparkr--${{ inputs.java }}-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" @@ -780,7 +780,7 @@ jobs: run: tar cjf site.tar.bz2 docs/_site - name: Upload documentation if: github.repository != 'apache/spark' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: site path: site.tar.bz2 @@ -933,13 +933,13 @@ jobs: spark.sql.join.forceApplyShuffledHashJoin=true - name: Upload test results to report if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-results-tpcds--${{ inputs.java }}-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" - name: Upload unit tests log files if: ${{ !success() }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit-tests-log-tpcds--${{ inputs.java }}-${{ inputs.hadoop }}-hive2.3 path: "**/target/unit-tests.log" @@ -1002,13 +1002,13 @@ jobs: ./dev/run-tests --parallelism 1 --modules docker-integration-tests --included-tags org.apache.spark.tags.DockerTest - name: Upload test results to report if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-results-docker-integration--${{ inputs.java }}-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" - name: Upload unit tests log files if: ${{ !success() }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit-tests-log-docker-integration--${{ inputs.java }}-${{ inputs.hadoop }}-hive2.3 path: "**/target/unit-tests.log" @@ -1083,7 +1083,7 @@ jobs: build/sbt -Phadoop-3 -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test" - name: Upload Spark on K8S integration tests log files if: ${{ !success() }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: spark-on-kubernetes-it-log path: "**/target/integration-tests.log" diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 078a380f0afea..3f19c76fd9616 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -200,13 +200,13 @@ jobs: rm -rf ~/.m2/repository/org/apache/spark - name: Upload test results to report if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-results-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }} path: "**/target/test-reports/*.xml" - name: Upload unit tests log files if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit-tests-log-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }} path: "**/target/unit-tests.log" From 6f06ba547534365e0cd8f3c8853e23387eae6d14 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Mon, 15 Jan 2024 14:24:37 +0800 Subject: [PATCH 2/3] add connect deps for connect pypy3 --- dev/infra/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 78814ace9b2e0..dd2abb8c8bb52 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -99,6 +99,7 @@ ARG BASIC_PIP_PKGS="numpy pyarrow>=14.0.0 six==1.16.0 pandas<=2.1.4 scipy plotly # Python deps for Spark Connect ARG CONNECT_PIP_PKGS="grpcio==1.59.3 grpcio-status==1.59.3 protobuf==4.25.1 googleapis-common-protos==1.56.4" +RUN pypy3 -m pip install $CONNECT_PIP_PKGS RUN python3.9 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS # Add torch as a testing dependency for TorchDistributor and DeepspeedTorchDistributor From 8c7f7423d36e97ba0d112aecddad7993ee192a11 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Mon, 15 Jan 2024 14:27:35 +0800 Subject: [PATCH 3/3] Revert "add connect deps for connect pypy3" This reverts commit 6f06ba547534365e0cd8f3c8853e23387eae6d14. --- dev/infra/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index dd2abb8c8bb52..78814ace9b2e0 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -99,7 +99,6 @@ ARG BASIC_PIP_PKGS="numpy pyarrow>=14.0.0 six==1.16.0 pandas<=2.1.4 scipy plotly # Python deps for Spark Connect ARG CONNECT_PIP_PKGS="grpcio==1.59.3 grpcio-status==1.59.3 protobuf==4.25.1 googleapis-common-protos==1.56.4" -RUN pypy3 -m pip install $CONNECT_PIP_PKGS RUN python3.9 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS # Add torch as a testing dependency for TorchDistributor and DeepspeedTorchDistributor