diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8649ec9730e..2b954320d0ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,337 +39,6 @@ concurrency: cancel-in-progress: true jobs: - maven-checks: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - java-version: - - 17 - timeout-minutes: 45 - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # checkout all commits, as the build result depends on `git describe` equivalent - - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: ${{ matrix.java-version }} - cache: 'maven' - - name: Configure Problem Matchers - run: echo "::add-matcher::.github/problem-matcher.json" - - name: Maven Checks - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $RETRY $MAVEN clean verify -B --strict-checksums -V -T C1 -DskipTests -P ci -pl '!:trino-server-rpm' - - name: Test Server RPM - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $RETRY bash -c "$MAVEN verify -B --strict-checksums -P ci -pl :trino-server-rpm || find core/trino-server-rpm/ -exec ls -ald {} +" - - name: Clean Maven Output - run: $MAVEN clean -pl '!:trino-server,!:trino-cli' - - uses: docker/setup-qemu-action@v1 - with: - platforms: arm64,ppc64le - - name: Test Docker Image - run: core/docker/build.sh - - error-prone-checks: - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB - - name: Fetch base ref to find merge-base for GIB - run: .github/bin/git-fetch-base-ref.sh - - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: 17 - cache: 'maven' - - name: Configure Problem Matchers - run: echo "::add-matcher::.github/problem-matcher.json" - - name: Maven Package - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $RETRY $MAVEN clean package ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -pl '!:trino-docs,!:trino-server,!:trino-server-rpm' - - name: Error Prone Checks - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - # Run Error Prone on one module with a retry to ensure all runtime dependencies are fetched - $RETRY $MAVEN ${MAVEN_TEST} -T C1 clean verify -DskipTests -P gib,errorprone-compiler -am -pl ':trino-spi' - # The main Error Prone run - $MAVEN ${MAVEN_TEST} -T C1 clean verify -DskipTests -P gib,errorprone-compiler \ - -pl '!:trino-docs,!:trino-server,!:trino-server-rpm' - - web-ui-checks: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # checkout all commits: it's not needed here, but it's needed almost always, so let's do this for completeness - - name: Web UI Checks - run: core/trino-main/bin/check_webui.sh - - test-jdbc-compatibility: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # checkout tags so version in Manifest is set properly - - name: Fetch base ref to find merge-base for GIB - run: .github/bin/git-fetch-base-ref.sh - - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: 17 - cache: 'maven' - - name: Configure Problem Matchers - run: echo "::add-matcher::.github/problem-matcher.json" - - name: Maven Install - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -pl '!:trino-docs,!:trino-server,!:trino-server-rpm' - - name: Test old JDBC vs current server - run: | - if [ ! -f gib-impacted.log ] || grep -q testing/trino-test-jdbc-compatibility-old-driver gib-impacted.log; then - testing/trino-test-jdbc-compatibility-old-driver/bin/run_tests.sh - fi - - name: Test current JDBC vs old server - if: always() - run: | - if [ ! -f gib-impacted.log ] || grep -q testing/trino-test-jdbc-compatibility-old-server gib-impacted.log; then - $MAVEN test ${MAVEN_TEST} -pl :trino-test-jdbc-compatibility-old-server - fi - - name: Upload test results - uses: actions/upload-artifact@v3 - # Upload all test reports only on failure, because the artifacts are large - if: failure() - with: - name: result ${{ github.job }} - path: | - **/target/surefire-reports - **/target/checkstyle-* - - name: Upload test report - uses: actions/upload-artifact@v3 - # Always upload the test report for the annotate.yml workflow, - # but only the single XML file to keep the artifact small - if: always() - with: - # Name prefix is checked in the `Annotate checks` workflow - name: test report ${{ github.job }} - path: | - **/surefire-reports/TEST-*.xml - retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} - - name: Clean local Maven repo - if: steps.cache.outputs.cache-hit != 'true' - run: rm -rf ~/.m2/repository - - hive-tests: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - config: - - config-empty - - config-hdp3 - # TODO: config-cdh5 - # TODO: config-apache-hive3 - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB - - name: Fetch base ref to find merge-base for GIB - run: .github/bin/git-fetch-base-ref.sh - - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: 17 - cache: 'maven' - - name: Configure Problem Matchers - run: echo "::add-matcher::.github/problem-matcher.json" - - name: Install Hive Module - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -am -pl :trino-hive-hadoop2 - - name: Run Hive Tests - run: | - source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && - plugin/trino-hive-hadoop2/bin/run_hive_tests.sh - - name: Run Hive S3 Tests - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESSKEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETKEY }} - S3_BUCKET: "presto-ci-test" - S3_BUCKET_ENDPOINT: "s3.us-east-2.amazonaws.com" - run: | - if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then - source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && - plugin/trino-hive-hadoop2/bin/run_hive_s3_tests.sh - fi - - name: Run Hive Glue Tests - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESSKEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETKEY }} - AWS_REGION: us-east-2 - run: | - if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then - $MAVEN test ${MAVEN_TEST} -pl :trino-hive -P test-hive-glue - fi - - name: Run Hive Azure ABFS Access Key Tests - if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage - env: - ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }} - ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }} - ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }} - run: | - if [ "${ABFS_CONTAINER}" != "" ]; then - source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && - plugin/trino-hive-hadoop2/bin/run_hive_abfs_access_key_tests.sh - fi - - name: Run Hive Azure ABFS OAuth Tests - if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage - env: - ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }} - ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }} - ABFS_OAUTH_ENDPOINT: ${{ secrets.AZURE_ABFS_OAUTH_ENDPOINT }} - ABFS_OAUTH_CLIENTID: ${{ secrets.AZURE_ABFS_OAUTH_CLIENTID }} - ABFS_OAUTH_SECRET: ${{ secrets.AZURE_ABFS_OAUTH_SECRET }} - run: | - if [ -n "$ABFS_CONTAINER" ]; then - source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && - plugin/trino-hive-hadoop2/bin/run_hive_abfs_oauth_tests.sh - fi - - name: Run Hive Azure WASB Tests - if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage - env: - WASB_CONTAINER: ${{ secrets.AZURE_WASB_CONTAINER }} - WASB_ACCOUNT: ${{ secrets.AZURE_WASB_ACCOUNT }} - WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESSKEY }} - run: | - if [ "${WASB_CONTAINER}" != "" ]; then - source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && - plugin/trino-hive-hadoop2/bin/run_hive_wasb_tests.sh - fi - - name: Run Hive Azure ADL Tests - if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage - env: - ADL_NAME: ${{ secrets.AZURE_ADL_NAME }} - ADL_CLIENT_ID: ${{ secrets.AZURE_ADL_CLIENTID }} - ADL_CREDENTIAL: ${{ secrets.AZURE_ADL_CREDENTIAL }} - ADL_REFRESH_URL: ${{ secrets.AZURE_ADL_REFRESHURL }} - run: | - if [ "${ADL_NAME}" != "" ]; then - source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && - plugin/trino-hive-hadoop2/bin/run_hive_adl_tests.sh - fi - - name: Run Hive Alluxio Tests - run: | - source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && - plugin/trino-hive-hadoop2/bin/run_hive_alluxio_tests.sh - - name: Upload test results - uses: actions/upload-artifact@v3 - # Upload all test reports only on failure, because the artifacts are large - if: failure() - with: - name: result ${{ github.job }} - path: | - **/target/surefire-reports - **/target/checkstyle-* - - name: Upload test report - uses: actions/upload-artifact@v3 - # Always upload the test report for the annotate.yml workflow, - # but only the single XML file to keep the artifact small - if: always() - with: - # Name prefix is checked in the `Annotate checks` workflow - name: test report ${{ github.job }} (${{ matrix.config }}) - path: | - **/surefire-reports/TEST-*.xml - retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} - - name: Clean local Maven repo - if: steps.cache.outputs.cache-hit != 'true' - run: rm -rf ~/.m2/repository - - test-other-modules: - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB - - name: Fetch base ref to find merge-base for GIB - run: .github/bin/git-fetch-base-ref.sh - - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: 17 - cache: 'maven' - - name: Configure Problem Matchers - run: echo "::add-matcher::.github/problem-matcher.json" - - name: Maven Install - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -pl '!:trino-docs,!:trino-server,!:trino-server-rpm' - - name: Maven Tests - run: | - $MAVEN test ${MAVEN_TEST} -pl ' - !:trino-main, - !:trino-tests, - !:trino-faulttolerant-tests, - !:trino-raptor-legacy, - !:trino-accumulo, - !:trino-cassandra, - !:trino-clickhouse, - !:trino-delta-lake, - !:trino-hive, - !:trino-elasticsearch, - !:trino-mongodb, - !:trino-kafka, - !:trino-pinot, - !:trino-redis, - !:trino-mysql, - !:trino-postgresql, - !:trino-sqlserver, - !:trino-mariadb, - !:trino-oracle, - !:trino-kudu, - !:trino-druid, - !:trino-iceberg, - !:trino-phoenix5, - !:trino-jdbc,!:trino-base-jdbc,!:trino-thrift,!:trino-memory, - !:trino-docs,!:trino-server,!:trino-server-rpm, - !:trino-test-jdbc-compatibility-old-server, - !:trino-singlestore, - !:trino-bigquery' - - name: Upload test results - uses: actions/upload-artifact@v3 - # Upload all test reports only on failure, because the artifacts are large - if: failure() - with: - name: result ${{ github.job }} - path: | - **/target/surefire-reports - **/target/checkstyle-* - - name: Upload test report - uses: actions/upload-artifact@v3 - # Always upload the test report for the annotate.yml workflow, - # but only the single XML file to keep the artifact small - if: always() - with: - # Name prefix is checked in the `Annotate checks` workflow - name: test report ${{ github.job }} - path: | - **/surefire-reports/TEST-*.xml - retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} - - name: Clean local Maven repo - if: steps.cache.outputs.cache-hit != 'true' - run: rm -rf ~/.m2/repository - build-test-matrix: runs-on: ubuntu-latest outputs: @@ -395,37 +64,56 @@ jobs: touch gib-impacted.log cat < .github/test-matrix.yaml include: - - { modules: core/trino-main } - - { modules: testing/trino-tests } - - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-hive-1 } - - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-hive-2 } - - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-delta } - - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-iceberg } - - { modules: plugin/trino-raptor-legacy } - - { modules: plugin/trino-accumulo } - - { modules: plugin/trino-cassandra } - - { modules: plugin/trino-clickhouse } - - { modules: plugin/trino-delta-lake } - - { modules: plugin/trino-hive } - - { modules: plugin/trino-hive, profile: test-parquet } - - { modules: plugin/trino-elasticsearch } - - { modules: plugin/trino-elasticsearch, profile: test-stats } - - { modules: plugin/trino-mongodb } - - { modules: plugin/trino-kafka } - - { modules: plugin/trino-pinot } - - { modules: plugin/trino-redis } - - { modules: plugin/trino-mysql } - - { modules: plugin/trino-postgresql } - - { modules: plugin/trino-sqlserver } - - { modules: plugin/trino-singlestore } - - { modules: plugin/trino-mariadb } - - { modules: plugin/trino-oracle } - - { modules: plugin/trino-kudu } - - { modules: plugin/trino-druid } - - { modules: plugin/trino-iceberg } - - { modules: plugin/trino-phoenix5 } - - { modules: [ client/trino-jdbc, plugin/trino-base-jdbc, plugin/trino-thrift, plugin/trino-memory ] } - - { modules: plugin/trino-bigquery } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } + - { modules: plugin/trino-cassandra } EOF ./.github/bin/build-matrix-from-impacted.py -v -i gib-impacted.log -m .github/test-matrix.yaml -o matrix.json echo "Matrix: $(jq '.' matrix.json)" @@ -539,245 +227,3 @@ jobs: - name: Clean local Maven repo if: steps.cache.outputs.cache-hit != 'true' run: rm -rf ~/.m2/repository - - build-pt: - runs-on: ubuntu-latest - outputs: - have_azure_secrets: ${{ steps.check-secrets.outputs.have_azure_secrets }} - have_databricks_secrets: ${{ steps.check-databricks-secrets.outputs.have_databricks_secrets }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB - - name: Fetch base ref to find merge-base for GIB - run: .github/bin/git-fetch-base-ref.sh - - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: 17 - cache: 'maven' - - name: Check secrets - run: | - if [[ "${{ secrets.AZURE_ABFS_CONTAINER }}" != "" && \ - "${{ secrets.AZURE_ABFS_ACCOUNT }}" != "" && \ - "${{ secrets.AZURE_ABFS_ACCESSKEY }}" != "" - ]]; \ - then - echo "Secrets to run product tests were configured in the repo" - echo "::set-output name=have_azure_secrets::true" - else - echo "Secrets to run product tests were not configured in the repo" - echo "::set-output name=have_azure_secrets::false" - fi - id: check-secrets - - name: Check Delta Databricks secrets - id: check-databricks-secrets - run: | - if [[ "${{ secrets.DATABRICKS_TOKEN }}" != "" ]]; \ - then - echo "Secrets to run Delta Databricks product tests were configured in the repo" - echo "::set-output name=have_databricks_secrets::true" - else - echo "Secrets to run Delta Databricks product tests were not configured in the repo" - echo "::set-output name=have_databricks_secrets::false" - fi - - name: Maven Install - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server-rpm' - - name: Map impacted plugins to features - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $MAVEN validate ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -pl '!:trino-docs,!:trino-server-rpm' - # GIB doesn't run on master, so make sure the file always exist - touch gib-impacted.log - testing/trino-plugin-reader/target/trino-plugin-reader-*-executable.jar -i gib-impacted.log -p core/trino-server/target/trino-server-*-hardlinks/plugin > impacted-features.log - echo "Impacted plugin features:" - cat impacted-features.log - - name: Product tests artifact - uses: actions/upload-artifact@v3 - with: - name: product tests and server tarball - path: | - core/trino-server/target/*.tar.gz - impacted-features.log - testing/trino-product-tests-launcher/target/*-executable.jar - testing/trino-product-tests/target/*-executable.jar - client/trino-cli/target/*-executable.jar - retention-days: 1 - - name: Clean local Maven repo - if: steps.cache.outputs.cache-hit != 'true' - run: rm -rf ~/.m2/repository - - pt: - runs-on: ubuntu-latest - # explicitly define the name to avoid adding the value of the `ignore exclusion if` matrix item - name: pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }}) - strategy: - fail-fast: false - matrix: - config: - - default - - hdp3 - - cdh5 - # TODO: config-apache-hive3 - suite: - - suite-1 - - suite-2 - - suite-3 - # suite-4 does not exist - - suite-5 - - suite-azure - - suite-delta-lake-databricks - exclude: - - config: cdh5 - ignore exclusion if: >- - ${{ github.event_name != 'pull_request' - || contains(github.event.pull_request.labels.*.name, 'tests:all') - || contains(github.event.pull_request.labels.*.name, 'tests:hive') - }} - - - config: default - ignore exclusion if: >- - ${{ github.event_name != 'pull_request' - || github.event.pull_request.head.repo.full_name == github.repository - || contains(github.event.pull_request.labels.*.name, 'tests:all') - || contains(github.event.pull_request.labels.*.name, 'tests:hive') - }} - - - suite: suite-azure - config: default - - suite: suite-azure - config: cdh5 - - suite: suite-azure - ignore exclusion if: >- - ${{ needs.build-pt.outputs.have_azure_secrets == 'true' }} - - - suite: suite-delta-lake-databricks - config: cdh5 - - suite: suite-delta-lake-databricks - config: hdp3 - - suite: suite-delta-lake-databricks - ignore exclusion if: >- - ${{ needs.build-pt.outputs.have_databricks_secrets == 'true' }} - - ignore exclusion if: - # Do not use this property outside of the matrix configuration. - # - # This is added to all matrix entries so they may be conditionally - # excluded by adding them to the excludes list with a GHA expression - # for this property. - # - If the expression evaluates to true, it will never match the a - # actual value of the property, and will therefore not be excluded. - # - If the expression evaluates to false, it will match the actual - # value of the property, and the exclusion will apply normally. - - false - include: - # this suite is not meant to be run with different configs - - config: default - suite: suite-6-non-generic - # this suite is not meant to be run with different configs - - config: default - suite: suite-7-non-generic - # this suite is not meant to be run with different configs - - config: default - suite: suite-8-non-generic - # this suite is not meant to be run with different configs - - config: default - suite: suite-tpcds - # this suite is not meant to be run with different configs - - config: default - suite: suite-oauth2 - # this suite is not meant to be run with different configs - - config: default - suite: suite-ldap - # this suite is not meant to be run with different configs - - config: default - suite: suite-compatibility - # this suite is designed specifically for apache-hive3. TODO remove the suite once we can run all regular tests on apache-hive3. - - config: apache-hive3 - suite: suite-hms-only - # this suite is not meant to be run with different configs - - config: default - suite: suite-all - # this suite is not meant to be run with different configs - - config: default - suite: suite-delta-lake-oss - # PT Launcher's timeout defaults to 2h, add some margin - timeout-minutes: 130 - needs: build-pt - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # checkout all commits, as the build result depends on `git describe` equivalent - - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: 17 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - product-tests: - - 'testing/trino-product-tests*/**' - - 'testing/trino-testing-services/**' - # run all tests when there are any changes in the trino-server Maven module - # because it uses Provisio to determine its dependencies - - 'core/trino-server/**' - - '.github/**' - - name: Product tests artifact - uses: actions/download-artifact@v3 - with: - name: product tests and server tarball - - name: Fix artifact permissions - run: | - find . -type f -name \*-executable.jar -exec chmod 0777 {} \; - - name: Enable impact analysis - # don't enable this on pushes to master and in PRs in the main repository (not from forks) - # because these are most often used to run all tests with additional secrets - if: | - steps.filter.outputs.product-tests == 'false' && - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository && - !contains(github.event.pull_request.labels.*.name, 'tests:all') && - !contains(github.event.pull_request.labels.*.name, 'tests:all-product') - run: echo "PTL_OPTS=--impacted-features impacted-features.log" >> $GITHUB_ENV - - name: Product Tests - env: - ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }} - ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }} - ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }} - S3_BUCKET: trino-ci-test - AWS_REGION: us-east-2 - DATABRICKS_AWS_ACCESS_KEY_ID: ${{ secrets.DATABRICKS_AWS_ACCESS_KEY_ID }} - DATABRICKS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DATABRICKS_AWS_SECRET_ACCESS_KEY }} - DATABRICKS_73_JDBC_URL: ${{ secrets.DATABRICKS_73_JDBC_URL }} - DATABRICKS_91_JDBC_URL: ${{ secrets.DATABRICKS_91_JDBC_URL }} - DATABRICKS_104_JDBC_URL: ${{ secrets.DATABRICKS_104_JDBC_URL }} - DATABRICKS_LOGIN: token - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - run: | - testing/bin/ptl suite run \ - --suite ${{ matrix.suite }} \ - --config config-${{ matrix.config }} \ - ${PTL_OPTS:-} \ - --bind=off --logs-dir logs/ --timeout 2h - - name: Upload test logs and results - uses: actions/upload-artifact@v3 - # Upload all test reports only on failure, because the artifacts are large - if: failure() - with: - name: result pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }}) - path: | - testing/trino-product-tests/target/* - logs/* - - name: Upload test report - uses: actions/upload-artifact@v3 - # Always upload the test report for the annotate.yml workflow, - # but only the single XML file to keep the artifact small - if: always() - with: - # Name prefix is checked in the `Annotate checks` workflow - name: test report pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }}) - path: testing/trino-product-tests/target/reports/**/testng-results.xml - retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} diff --git a/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/CassandraServer.java b/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/CassandraServer.java index a82f3563ebad..eda0d71e621a 100644 --- a/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/CassandraServer.java +++ b/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/CassandraServer.java @@ -20,6 +20,7 @@ import com.datastax.oss.driver.api.core.config.ProgrammaticDriverConfigLoaderBuilder; import com.datastax.oss.driver.api.core.cql.ResultSet; import com.datastax.oss.driver.api.core.cql.Row; +import com.google.common.collect.ImmutableMap; import com.google.common.io.Resources; import io.airlift.json.JsonCodec; import io.airlift.log.Logger; @@ -32,6 +33,7 @@ import java.net.InetSocketAddress; import java.nio.file.Path; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeoutException; import static com.datastax.oss.driver.api.core.config.DefaultDriverOption.CONTROL_CONNECTION_AGREEMENT_TIMEOUT; @@ -57,7 +59,7 @@ public class CassandraServer { private static Logger log = Logger.get(CassandraServer.class); - private static final int PORT = 9142; + private static final int PORT = 9042; private static final Duration REFRESH_SIZE_ESTIMATES_TIMEOUT = new Duration(1, MINUTES); @@ -67,17 +69,25 @@ public class CassandraServer public CassandraServer() throws Exception { - this("2.2"); + this("cassandra:2.2", ImmutableMap.of()); } - public CassandraServer(String cassandraVersion) + public CassandraServer(String imageName) + throws Exception + { + this(imageName, ImmutableMap.of()); + } + + public CassandraServer(String imageName, Map environmentVariables) throws Exception { log.info("Starting cassandra..."); - this.dockerContainer = new GenericContainer<>("cassandra:" + cassandraVersion) + this.dockerContainer = new GenericContainer<>(imageName) .withExposedPorts(PORT) - .withCopyFileToContainer(forHostPath(prepareCassandraYaml()), "/etc/cassandra/cassandra.yaml"); + .withCopyFileToContainer(forHostPath(prepareCassandraYaml()), "/etc/cassandra/cassandra.yaml") + .withEnv(environmentVariables) + .withStartupTimeout(java.time.Duration.ofMinutes(10)); this.dockerContainer.start(); ProgrammaticDriverConfigLoaderBuilder driverConfigLoaderBuilder = DriverConfigLoader.programmaticBuilder(); diff --git a/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestCassandraLatestConnectorSmokeTest.java b/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestCassandraLatestConnectorSmokeTest.java index 2109ce2cc262..e053d06af2ee 100644 --- a/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestCassandraLatestConnectorSmokeTest.java +++ b/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestCassandraLatestConnectorSmokeTest.java @@ -28,7 +28,7 @@ public class TestCassandraLatestConnectorSmokeTest protected QueryRunner createQueryRunner() throws Exception { - CassandraServer server = closeAfterClass(new CassandraServer("3.11.10")); + CassandraServer server = closeAfterClass(new CassandraServer("cassandra:3.11.10")); CassandraSession session = server.getSession(); createTestTables(session, KEYSPACE, Timestamp.from(TIMESTAMP_VALUE.toInstant())); return createCassandraQueryRunner(server, ImmutableMap.of(), ImmutableMap.of(), REQUIRED_TPCH_TABLES); diff --git a/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestDatastaxConnectorSmokeTest.java b/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestDatastaxConnectorSmokeTest.java new file mode 100644 index 000000000000..f9d15d288354 --- /dev/null +++ b/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestDatastaxConnectorSmokeTest.java @@ -0,0 +1,41 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.cassandra; + +import com.google.common.collect.ImmutableMap; +import io.trino.testing.QueryRunner; + +import java.sql.Timestamp; +import java.util.Map; + +import static io.trino.plugin.cassandra.CassandraQueryRunner.createCassandraQueryRunner; +import static io.trino.plugin.cassandra.CassandraTestingUtils.createTestTables; + +public class TestDatastaxConnectorSmokeTest + extends BaseCassandraConnectorSmokeTest +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + CassandraServer server = closeAfterClass(new CassandraServer( + "datastax/dse-server:6.8.25", + Map.of( + "DS_LICENSE", "accept", + "DC", "datacenter1"))); + CassandraSession session = server.getSession(); + createTestTables(session, KEYSPACE, Timestamp.from(TIMESTAMP_VALUE.toInstant())); + return createCassandraQueryRunner(server, ImmutableMap.of(), ImmutableMap.of(), REQUIRED_TPCH_TABLES); + } +} diff --git a/plugin/trino-cassandra/src/test/resources/cu-cassandra.yaml b/plugin/trino-cassandra/src/test/resources/cu-cassandra.yaml index 790a6d78a04c..bcb669659316 100644 --- a/plugin/trino-cassandra/src/test/resources/cu-cassandra.yaml +++ b/plugin/trino-cassandra/src/test/resources/cu-cassandra.yaml @@ -261,7 +261,7 @@ listen_address: 127.0.0.1 start_native_transport: true # port for the CQL native transport to listen for clients on -native_transport_port: 9142 +native_transport_port: 9042 # Whether to start the thrift rpc server. start_rpc: false