From 049550a6f8826d3e5f5e443b528755f340b0f6cd Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 6 Dec 2023 16:34:06 +0800 Subject: [PATCH 01/10] init --- .github/workflows/build_and_test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d58356ec1c5dc..52606a729e7b1 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -201,6 +201,7 @@ jobs: SKIP_UNIDOC: true SKIP_MIMA: true SKIP_PACKAGING: true + BRANCH: ${{ inputs.branch }} steps: - name: Checkout Spark repository uses: actions/checkout@v4 @@ -268,6 +269,9 @@ jobs: export TERM=vt100 # Hive "other tests" test needs larger metaspace size based on experiment. if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" == "org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi + if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "BRANCH" != "master" ]]; then + MODULES_TO_TEST=${MODULES_TO_TEST//streaming-kinesis-asl, /} + fi export SERIAL_SBT_TESTS=1 ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS" - name: Upload test results to report From b0b2c3db78f47cafc35dee781f4cfa9c9d367c0d Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 6 Dec 2023 16:40:51 +0800 Subject: [PATCH 02/10] init --- .github/workflows/build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 52606a729e7b1..01a74d6aa39d9 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -269,6 +269,7 @@ jobs: export TERM=vt100 # Hive "other tests" test needs larger metaspace size based on experiment. if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" == "org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi + // SPARK-46283: should delete the following env replacement after SPARK 3.x EOL if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "BRANCH" != "master" ]]; then MODULES_TO_TEST=${MODULES_TO_TEST//streaming-kinesis-asl, /} fi From b7acf54fafb1204f1a7f1655a15db2ba86feaecb Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 6 Dec 2023 16:42:12 +0800 Subject: [PATCH 03/10] for test --- .github/workflows/build_and_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 01a74d6aa39d9..85befd7815933 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -270,9 +270,10 @@ jobs: # Hive "other tests" test needs larger metaspace size based on experiment. if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" == "org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi // SPARK-46283: should delete the following env replacement after SPARK 3.x EOL - if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "BRANCH" != "master" ]]; then + if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "BRANCH" != "master-1" ]]; then MODULES_TO_TEST=${MODULES_TO_TEST//streaming-kinesis-asl, /} fi + echo $MODULES_TO_TEST export SERIAL_SBT_TESTS=1 ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS" - name: Upload test results to report From f9e22e058ac4583e89a7fb4ac2daa3f2db1bcb36 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 6 Dec 2023 16:55:15 +0800 Subject: [PATCH 04/10] Revert "for test" This reverts commit b7acf54fafb1204f1a7f1655a15db2ba86feaecb. --- .github/workflows/build_and_test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 85befd7815933..01a74d6aa39d9 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -270,10 +270,9 @@ jobs: # Hive "other tests" test needs larger metaspace size based on experiment. if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" == "org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi // SPARK-46283: should delete the following env replacement after SPARK 3.x EOL - if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "BRANCH" != "master-1" ]]; then + if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "BRANCH" != "master" ]]; then MODULES_TO_TEST=${MODULES_TO_TEST//streaming-kinesis-asl, /} fi - echo $MODULES_TO_TEST export SERIAL_SBT_TESTS=1 ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS" - name: Upload test results to report From ae40716fb88b09ec892a29234746368aa6781a50 Mon Sep 17 00:00:00 2001 From: YangJie Date: Wed, 6 Dec 2023 18:37:35 +0800 Subject: [PATCH 05/10] Update .github/workflows/build_and_test.yml Co-authored-by: Hyukjin Kwon --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 01a74d6aa39d9..347ebd4b323c0 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -270,7 +270,7 @@ jobs: # Hive "other tests" test needs larger metaspace size based on experiment. if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" == "org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi // SPARK-46283: should delete the following env replacement after SPARK 3.x EOL - if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "BRANCH" != "master" ]]; then + if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "${{ inputs.branch }}" != "master" ]]; then MODULES_TO_TEST=${MODULES_TO_TEST//streaming-kinesis-asl, /} fi export SERIAL_SBT_TESTS=1 From a6dee9f10556bf62777136b1e6c18c20a346b2a2 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 6 Dec 2023 18:38:36 +0800 Subject: [PATCH 06/10] remove BRANCH --- .github/workflows/build_and_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 347ebd4b323c0..c039a89f93026 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -201,7 +201,6 @@ jobs: SKIP_UNIDOC: true SKIP_MIMA: true SKIP_PACKAGING: true - BRANCH: ${{ inputs.branch }} steps: - name: Checkout Spark repository uses: actions/checkout@v4 From 0b0097561a36a8ad28c75ffff212983c13469744 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 6 Dec 2023 18:41:12 +0800 Subject: [PATCH 07/10] test echo --- .github/workflows/build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index c039a89f93026..f86cc5b0a9466 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -272,6 +272,7 @@ jobs: if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "${{ inputs.branch }}" != "master" ]]; then MODULES_TO_TEST=${MODULES_TO_TEST//streaming-kinesis-asl, /} fi + echo $MODULES_TO_TEST export SERIAL_SBT_TESTS=1 ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS" - name: Upload test results to report From 1a68951f3dec8f820b3c28ef5ddeb40da90cb3ce Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 6 Dec 2023 18:41:49 +0800 Subject: [PATCH 08/10] test echo --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index f86cc5b0a9466..767356650b201 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -269,7 +269,7 @@ jobs: # Hive "other tests" test needs larger metaspace size based on experiment. if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" == "org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi // SPARK-46283: should delete the following env replacement after SPARK 3.x EOL - if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "${{ inputs.branch }}" != "master" ]]; then + if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "${{ inputs.branch }}" != "master-1" ]]; then MODULES_TO_TEST=${MODULES_TO_TEST//streaming-kinesis-asl, /} fi echo $MODULES_TO_TEST From e120b8cebff06ddfe649b8efef6b9872cc3387ce Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 6 Dec 2023 18:42:10 +0800 Subject: [PATCH 09/10] revert --- .github/workflows/build_and_test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 767356650b201..c039a89f93026 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -269,10 +269,9 @@ jobs: # Hive "other tests" test needs larger metaspace size based on experiment. if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" == "org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi // SPARK-46283: should delete the following env replacement after SPARK 3.x EOL - if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "${{ inputs.branch }}" != "master-1" ]]; then + if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "${{ inputs.branch }}" != "master" ]]; then MODULES_TO_TEST=${MODULES_TO_TEST//streaming-kinesis-asl, /} fi - echo $MODULES_TO_TEST export SERIAL_SBT_TESTS=1 ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS" - name: Upload test results to report From a026530ad01cd79c6901bd8c4236af742520fb8d Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 6 Dec 2023 23:36:20 +0800 Subject: [PATCH 10/10] change to match start with branch-3 --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index c039a89f93026..920ac6a38c872 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -269,7 +269,7 @@ jobs: # Hive "other tests" test needs larger metaspace size based on experiment. if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" == "org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi // SPARK-46283: should delete the following env replacement after SPARK 3.x EOL - if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "${{ inputs.branch }}" != "master" ]]; then + if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "${{ inputs.branch }}" =~ ^branch-3 ]]; then MODULES_TO_TEST=${MODULES_TO_TEST//streaming-kinesis-asl, /} fi export SERIAL_SBT_TESTS=1