From 8e45bc8a9079fdbae7ac4d70a67b9441e22fd13d Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Tue, 6 Oct 2020 23:36:19 +0900 Subject: [PATCH 1/4] Added GitHub Actions job for building Spark with SBT for Scala 2.13. --- .github/workflows/build_and_test.yml | 24 +++++++++++++++++++ .../kinesis/KinesisBackedBlockRDD.scala | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 667371dacf5dc..35b27a7b62dfa 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -319,3 +319,27 @@ jobs: ./dev/change-scala-version.sh 2.13 ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=11 -Pscala-2.13 install rm -rf ~/.m2/repository/org/apache/spark + + scala-213-sbt: + name: Scala 2.13 build with SBT + runs-on: ubuntu-latest + steps: + - name: Checkout Spark repository + uses: actions/checkout@v2 + - name: Cache Ivy local repository + uses: actions/cache@v2 + with: + path: ~/.ivy2/cache + key: scala-213-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + restore-keys: | + scala-213-ivy- + - name: Install Java 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build with SBT + run: | + mkdir -p ~/.ivy2 + ./dev/change-scala-version.sh 2.13 + ./build/sbt -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Djava.version=11 -Pscala-2.13 publishLocal + rm -rf ~/.ivy2/cache/org.apache.spark diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala index 5072b3ae21d87..ab55d545770e9 100644 --- a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala +++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala @@ -91,7 +91,7 @@ class KinesisBackedBlockRDD[T: ClassTag]( require(_blockIds.length == arrayOfseqNumberRanges.length, "Number of blockIds is not equal to the number of sequence number ranges") - override def isValid(): Boolean = true + override def isValid: Boolean = true override def getPartitions: Array[Partition] = { Array.tabulate(_blockIds.length) { i => From 5a22c38e51bc42a887e9ab64734159cf9b432bac Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Wed, 7 Oct 2020 00:04:08 +0900 Subject: [PATCH 2/4] Fix build_and_test.yml to consider .m2, test:compile and local cache for ivy2. --- .github/workflows/build_and_test.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 35b27a7b62dfa..c4d34aafff059 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -339,7 +339,9 @@ jobs: java-version: 11 - name: Build with SBT run: | - mkdir -p ~/.ivy2 + mkdir -p ~/.ivy2/local + mkdir -p ~/.m2 ./dev/change-scala-version.sh 2.13 - ./build/sbt -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Djava.version=11 -Pscala-2.13 publishLocal - rm -rf ~/.ivy2/cache/org.apache.spark + ./build/sbt -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Djava.version=11 -Pscala-2.13 test:compile publishLocal + rm -rf ~/.ivy2/local/org.apache.spark + rm -rf ~/.m2/repository/org/apache/spark From 47a48063bbe56d5c84b4dc1fe7a871ef14de4ee2 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Fri, 9 Oct 2020 10:19:42 +0900 Subject: [PATCH 3/4] Replaced Maven test with SBT test for Scala 2.13. --- .github/workflows/build_and_test.yml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index c4d34aafff059..a046762369114 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -294,32 +294,6 @@ jobs: ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=11 install rm -rf ~/.m2/repository/org/apache/spark - scala-213: - name: Scala 2.13 build - runs-on: ubuntu-latest - steps: - - name: Checkout Spark repository - uses: actions/checkout@v2 - - name: Cache Maven local repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: scala-213-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - scala-213-maven- - - name: Install Java 11 - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Build with Maven - run: | - export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" - export MAVEN_CLI_OPTS="--no-transfer-progress" - mkdir -p ~/.m2 - ./dev/change-scala-version.sh 2.13 - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=11 -Pscala-2.13 install - rm -rf ~/.m2/repository/org/apache/spark - scala-213-sbt: name: Scala 2.13 build with SBT runs-on: ubuntu-latest From 30b2ddd0d69434379075488418551eb3400080af Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Fri, 9 Oct 2020 18:08:35 +0900 Subject: [PATCH 4/4] Changed the build goal to compile and test:compile. --- .github/workflows/build_and_test.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a046762369114..f5b9a897240cf 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -294,8 +294,8 @@ jobs: ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=11 install rm -rf ~/.m2/repository/org/apache/spark - scala-213-sbt: - name: Scala 2.13 build with SBT + scala-213: + name: Scala 2.13 build runs-on: ubuntu-latest steps: - name: Checkout Spark repository @@ -313,9 +313,5 @@ jobs: java-version: 11 - name: Build with SBT run: | - mkdir -p ~/.ivy2/local - mkdir -p ~/.m2 ./dev/change-scala-version.sh 2.13 - ./build/sbt -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Djava.version=11 -Pscala-2.13 test:compile publishLocal - rm -rf ~/.ivy2/local/org.apache.spark - rm -rf ~/.m2/repository/org/apache/spark + ./build/sbt -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Djava.version=11 -Pscala-2.13 compile test:compile