diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a373b0e76e7a..136f1923548f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -100,6 +100,7 @@ jobs: \"lint\" : \"true\", \"k8s-integration-tests\" : \"true\", \"breaking-changes-buf\" : \"true\", + \"connect-maven\": \"true\", }" echo $precondition # For debugging # Remove `\n` to avoid "Invalid format" error @@ -728,6 +729,86 @@ jobs: ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=${JAVA_VERSION/-ea} install rm -rf ~/.m2/repository/org/apache/spark + connect-maven: + needs: precondition + if: fromJson(needs.precondition.outputs.required).connect-maven == 'true' + name: Test connect modules with Maven using Java ${{ matrix.java }} + strategy: + fail-fast: false + matrix: + java: + - ${{ inputs.java }} + runs-on: ubuntu-22.04 + steps: + - name: Checkout Spark repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + repository: apache/spark + ref: ${{ inputs.branch }} + - name: Sync the current branch with the latest in Apache Spark + if: github.repository != 'apache/spark' + run: | + git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty + - name: Cache Scala, SBT and Maven + uses: actions/cache@v3 + with: + path: | + build/apache-maven-* + build/scala-* + build/*.jar + ~/.sbt + key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} + restore-keys: | + build- + - name: Cache Maven local repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: java${{ matrix.java }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + java${{ matrix.java }}-maven- + - name: Install Java ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: ${{ matrix.java }} + - name: Build and Test with Maven + shell: 'script -q -e -c "bash {0}"' + run: | + # Fix for TTY related issues when launching the Ammonite REPL in tests. + export TERM=vt100 && script -qfc 'echo exit | amm -s' && rm typescript + set -e + export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" + export MAVEN_CLI_OPTS="--no-transfer-progress" + export JAVA_VERSION=${{ matrix.java }} + # It uses Maven's 'install' intentionally, see https://github.com/apache/spark/pull/26414. + # 1. Test with -Phive + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -Phive + # TODO(SPARK-43989): Enable connect server module testing after resolving SPARK-43646 + # ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} test -pl connector/connect/server -Phive + ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} test -pl connector/connect/client/jvm -Phive + # 2. Test without -Phive + # ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl assembly + # TODO(SPARK-43989): Enable connect server module testing after resolving SPARK-43646 + # ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} test -pl connector/connect/server + # ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} test -pl connector/connect/client/jvm + # rm -rf ~/.m2/repository/org/apache/spark + - name: Upload test results to report + if: always() + uses: actions/upload-artifact@v3 + with: + name: test-results-connect-maven-${{ matrix.java }} + path: "**/target/test-reports/*.xml" + - name: Upload unit tests log files + if: failure() + uses: actions/upload-artifact@v3 + with: + name: unit-tests-log-connect-maven-${{ matrix.java }} + path: "**/target/unit-tests.log" + scala-213: needs: precondition if: fromJson(needs.precondition.outputs.required).scala-213 == 'true' diff --git a/.github/workflows/build_branch33.yml b/.github/workflows/build_branch33.yml index 7ceafceb7180..52bb47fad437 100644 --- a/.github/workflows/build_branch33.yml +++ b/.github/workflows/build_branch33.yml @@ -45,5 +45,6 @@ jobs: "sparkr": "true", "tpcds-1g": "true", "docker-integration-tests": "true", - "lint" : "true" + "lint" : "true", + "connect-maven" : "false", }