Skip to content
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions .github/workflows/build_and_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ jobs:
\"lint\" : \"true\",
\"k8s-integration-tests\" : \"true\",
\"breaking-changes-buf\" : \"true\",
\"connect-maven\": \"true\",
}"
echo $precondition # For debugging
# Remove `\n` to avoid "Invalid format" error
Expand Down Expand Up @@ -728,6 +729,83 @@ jobs:
./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=${JAVA_VERSION/-ea} install
rm -rf ~/.m2/repository/org/apache/spark

connect-maven:
needs: precondition
if: fromJson(needs.precondition.outputs.required).connect-maven == 'true'
name: Test connect modules with Maven using Java ${{ matrix.java }}
strategy:
fail-fast: false
matrix:
java:
- ${{ inputs.java }}
runs-on: ubuntu-22.04
steps:
- name: Checkout Spark repository
uses: actions/checkout@v3
with:
fetch-depth: 0
repository: apache/spark
ref: ${{ inputs.branch }}
- name: Sync the current branch with the latest in Apache Spark
if: github.repository != 'apache/spark'
run: |
git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty
- name: Cache Scala, SBT and Maven
uses: actions/cache@v3
with:
path: |
build/apache-maven-*
build/scala-*
build/*.jar
~/.sbt
key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
restore-keys: |
build-
- name: Cache Maven local repository
uses: actions/cache@v3
with:
path: ~/.m2/repository
key: java${{ matrix.java }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
java${{ matrix.java }}-maven-
- name: Install Java ${{ matrix.java }}
uses: actions/setup-java@v3
with:
distribution: temurin
java-version: ${{ matrix.java }}
- name: Build and Test with Maven
shell: 'script -q -e -c "bash {0}"'
run: |
# Fix for TTY related issues when launching the Ammonite REPL in tests.
export TERM=vt100 && script -qfc 'echo exit | amm -s' && rm typescript
# `set -e` to make the exit status as expected due to use `script -q -e -c` to run the commands

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

another way is add a new script, maybe named dev/run-connect-maven-tests as follows:

#!/usr/bin/env bash

set -e


# Go to the Spark project root directory
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
cd "$FWDIR"
export SPARK_HOME=$FWDIR
echo "$SPARK_HOME"


if [[ -z "$JAVA_VERSION" ]]; then
  JAVA_VERSION=8
fi


export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
export MAVEN_CLI_OPTS="--no-transfer-progress"


# 1. Test with -Phive
# It uses Maven's 'install' intentionally, see https://github.com/apache/spark/pull/26414.
build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -Phive
build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} test -pl connector/connect/client/jvm -Phive

# 2. Test without -Phive
build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl assembly
build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} test -pl connector/connect/client/jvm

and run: can change to:

          # Fix for TTY related issues when launching the Ammonite REPL in tests.
          export TERM=vt100 && script -qfc 'echo exit | amm -s' && rm typescript
          export JAVA_VERSION=${{ matrix.java }}
          ./dev/run-connect-maven-tests
          TEST_RETCODE=$?
          rm -rf ~/.m2/repository/org/apache/spark
          exit $TEST_RETCODE

I tested it and it worked, but due to lack of this script in branch-3.4, we need to add "connect-maven" : "false" to build_branch34.yml when using script

set -e
export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
export MAVEN_CLI_OPTS="--no-transfer-progress"
export JAVA_VERSION=${{ matrix.java }}
# It uses Maven's 'install' intentionally, see https://github.com/apache/spark/pull/26414.
# 1. Test with -Phive
./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -Phive
./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} test -pl connector/connect/client/jvm -Phive
# 2. Test without -Phive
./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl assembly
./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} test -pl connector/connect/client/jvm
rm -rf ~/.m2/repository/org/apache/spark
- name: Upload test results to report
if: always()
uses: actions/upload-artifact@v3
with:
name: test-results-connect-maven-${{ matrix.java }}
path: "**/target/test-reports/*.xml"
- name: Upload unit tests log files
if: failure()
uses: actions/upload-artifact@v3
with:
name: unit-tests-log-connect-maven-${{ matrix.java }}
path: "**/target/unit-tests.log"

scala-213:
needs: precondition
if: fromJson(needs.precondition.outputs.required).scala-213 == 'true'
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/build_branch33.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,5 +45,6 @@ jobs:
"sparkr": "true",
"tpcds-1g": "true",
"docker-integration-tests": "true",
"lint" : "true"
"lint" : "true",
"connect-maven" : "false"
}