diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml deleted file mode 100644 index 176ee6e86db25..0000000000000 --- a/.github/workflows/cleanup.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: cleanup - -on: - schedule: - - cron: '*/5 * * * *' - -jobs: - cancel: - runs-on: ubuntu-latest - steps: - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: hive-tests.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: kudu.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: maven-checks.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: product-tests-basic-environment.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: product-tests-specific-environment.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: spark-integration.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: tests.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: web-ui-checks.yml diff --git a/.github/workflows/hive-tests.yml b/.github/workflows/hive-tests.yml index 73926a0be725f..b97a1f3cf24c4 100644 --- a/.github/workflows/hive-tests.yml +++ b/.github/workflows/hive-tests.yml @@ -31,12 +31,16 @@ jobs: - '!presto-docs/**' hive-tests: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-hive-tests-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-hive-tests-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -45,7 +49,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs @@ -75,12 +79,16 @@ jobs: fi hive-dockerized-tests: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 20 concurrency: - group: ${{ github.workflow }}-hive-dockerized-tests-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-hive-dockerized-tests-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -89,7 +97,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs diff --git a/.github/workflows/kudu.yml b/.github/workflows/kudu.yml index 1616df0f10d15..8d1c4151ef809 100644 --- a/.github/workflows/kudu.yml +++ b/.github/workflows/kudu.yml @@ -29,12 +29,16 @@ jobs: codechange: - '!presto-docs/**' kudu: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-kudu-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-kudu-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -43,7 +47,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml index ff7403f8f39e1..629239f8f3100 100644 --- a/.github/workflows/maven-checks.yml +++ b/.github/workflows/maven-checks.yml @@ -12,10 +12,14 @@ env: jobs: maven-checks: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest timeout-minutes: 45 concurrency: - group: ${{ github.workflow }}-maven-checks-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-maven-checks-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - name: Free Disk Space @@ -29,7 +33,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs diff --git a/.github/workflows/product-tests-basic-environment.yml b/.github/workflows/product-tests-basic-environment.yml index 26eb105b61794..5d44eba7bb1f2 100644 --- a/.github/workflows/product-tests-basic-environment.yml +++ b/.github/workflows/product-tests-basic-environment.yml @@ -29,12 +29,16 @@ jobs: codechange: - '!presto-docs/**' product-tests-basic-environment: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-product-tests-basic-environment-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-product-tests-basic-environment-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - name: Free Disk Space @@ -49,7 +53,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs @@ -58,4 +62,6 @@ jobs: export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl '!presto-docs,!presto-spark-package,!presto-spark-launcher,!presto-spark-testing,!presto-test-coverage' - name: Run Product Tests Basic Environment + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh multinode -x quarantine,big_query,storage_formats,profile_specific_tests,tpcds,cassandra,mysql_connector,postgresql_connector,mysql,kafka,avro diff --git a/.github/workflows/product-tests-specific-environment.yml b/.github/workflows/product-tests-specific-environment.yml index fc5647ebd2b17..0a29cec164958 100644 --- a/.github/workflows/product-tests-specific-environment.yml +++ b/.github/workflows/product-tests-specific-environment.yml @@ -29,12 +29,16 @@ jobs: codechange: - '!presto-docs/**' product-tests-specific-environment1: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-product-tests-specific-environment1-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-product-tests-specific-environment1-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - name: Free Disk Space @@ -49,7 +53,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs @@ -58,26 +62,40 @@ jobs: export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl '!presto-docs,!presto-spark-package,!presto-spark-launcher,!presto-spark-testing,!presto-test-coverage' - name: Product Tests Specific 1.1 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode -g hdfs_no_impersonation,avro - name: Product Tests Specific 1.2 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-no-impersonation -g hdfs_no_impersonation # temporarily disable this flaky run. see issue #20388 for details # - name: Product Tests Specific 1.3 # run: presto-product-tests/bin/run_on_docker.sh singlenode-hdfs-impersonation -g storage_formats,cli,hdfs_impersonation - name: Product Tests Specific 1.4 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation -g storage_formats,cli,hdfs_impersonation,authorization,hive_file_header - name: Product Tests Specific 1.5 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation-cross-realm -g storage_formats,cli,hdfs_impersonation - name: Product Tests Specific 1.6 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh multinode-tls-kerberos -g cli,group-by,join,tls product-tests-specific-environment2: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-product-tests-specific-environment2-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-product-tests-specific-environment2-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - name: Free Disk Space @@ -92,7 +110,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs @@ -101,19 +119,33 @@ jobs: export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl '!presto-docs,!presto-spark-package,!presto-spark-launcher,!presto-spark-testing,!presto-test-coverage' - name: Product Tests Specific 2.1 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-ldap -g ldap -x simba_jdbc - name: Product Tests Specific 2.2 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh multinode-tls -g smoke,cli,group-by,join,tls - name: Product Tests Specific 2.3 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-mysql -g mysql_connector,mysql - name: Product Tests Specific 2.4 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-postgresql -g postgresql_connector - name: Product Tests Specific 2.5 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-cassandra -g cassandra # temporarily disable this flaky run. see issue #20388 for details # - name: Product Tests Specific 2.6 # run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation-with-wire-encryption -g storage_formats,cli,hdfs_impersonation,authorization - name: Product Tests Specific 2.7 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-kafka -g kafka - name: Product Tests Specific 2.8 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-sqlserver -g sqlserver diff --git a/.github/workflows/singlestore-tests.yml b/.github/workflows/singlestore-tests.yml index 30cc28b08da87..cd4b2c91b7660 100644 --- a/.github/workflows/singlestore-tests.yml +++ b/.github/workflows/singlestore-tests.yml @@ -30,13 +30,17 @@ jobs: codechange: - '!presto-docs/**' singlestore-dockerized-tests: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 30 concurrency: - group: ${{ github.workflow }}-singlestore-dockerized-tests-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-singlestore-dockerized-tests-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -56,7 +60,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs diff --git a/.github/workflows/spark-integration.yml b/.github/workflows/spark-integration.yml index b193bfb4a3d03..005b18231c1b1 100644 --- a/.github/workflows/spark-integration.yml +++ b/.github/workflows/spark-integration.yml @@ -30,12 +30,16 @@ jobs: codechange: - '!presto-docs/**' spark-integration: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-spark-integration-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-spark-integration-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -44,7 +48,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs diff --git a/.github/workflows/test-other-modules.yml b/.github/workflows/test-other-modules.yml index 4fece0113970a..131a8f4a99ded 100644 --- a/.github/workflows/test-other-modules.yml +++ b/.github/workflows/test-other-modules.yml @@ -30,12 +30,16 @@ jobs: codechange: - '!presto-docs/**' test-other-modules: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-test-other-modules-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-test-other-modules-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -44,7 +48,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b5fe34883ad64..84f0d30790739 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,7 @@ jobs: strategy: fail-fast: false matrix: + java: [8, 17.0.13] modules: - ":presto-tests -P presto-tests-execution-memory" - ":presto-tests -P presto-tests-general" @@ -68,7 +69,7 @@ jobs: - ":presto-iceberg" timeout-minutes: 80 concurrency: - group: ${{ github.workflow }}-test-${{ matrix.modules }}-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-test-${{ matrix.modules }}-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -79,7 +80,7 @@ jobs: if: needs.changes.outputs.codechange == 'true' with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache if: needs.changes.outputs.codechange == 'true' diff --git a/.mvn/jvm.config b/.mvn/jvm.config index 608348df8499a..5abda81f42fca 100644 --- a/.mvn/jvm.config +++ b/.mvn/jvm.config @@ -1 +1 @@ --Xmx8192m +-Xmx8192m diff --git a/pom.xml b/pom.xml index e7f3d0dba6661..c15ee6ff24a24 100644 --- a/pom.xml +++ b/pom.xml @@ -56,7 +56,7 @@ 2.12.7 - 1.53 + 1.54 7.5 8.10.0 3.8.0 @@ -67,7 +67,7 @@ 1.9.17 313 1.7.32 - 2.3.1 + 3.9.0 0.11.0 30.0.1 2.3.1 @@ -1373,7 +1373,7 @@ org.pcollections pcollections - 2.1.2 + 4.0.2 @@ -2184,6 +2184,10 @@ com.google.inject.extensions guice-multibindings + + org.testng + testng + @@ -2196,6 +2200,10 @@ com.datastax.cassandra cassandra-driver-core + + org.testng + testng + @@ -2212,6 +2220,10 @@ org.slf4j slf4j-log4j12 + + org.testng + testng + @@ -2224,6 +2236,10 @@ com.datastax.cassandra cassandra-driver-core + + org.testng + testng + @@ -2718,6 +2734,46 @@ + + java17 + + 17 + + + + --add-opens java.base/java.io=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.lang.ref=ALL-UNNAMED + --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/java.net=ALL-UNNAMED + --add-opens java.base/java.nio=ALL-UNNAMED + --add-opens java.base/java.security=ALL-UNNAMED + --add-opens java.base/javax.security.auth=ALL-UNNAMED + --add-opens java.base/javax.security.auth.login=ALL-UNNAMED + --add-opens java.base/java.text=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.util.concurrent=ALL-UNNAMED + --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens java.base/java.util.regex=ALL-UNNAMED + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED + --add-opens java.base/sun.security.action=ALL-UNNAMED + --add-opens java.base/sun.security.util=ALL-UNNAMED + --add-opens java.base/jdk.internal.module=ALL-UNNAMED + --add-opens java.base/java.lang.module=ALL-UNNAMED + --add-opens java.base/java.util.jar=ALL-UNNAMED + --add-opens java.base/java.util.zip=ALL-UNNAMED + --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED + --add-opens java.base/sun.reflect.annotation=ALL-UNNAMED + --add-opens java.base/sun.nio.fs=ALL-UNNAMED + --add-opens java.base/jdk.internal.ref=ALL-UNNAMED + --add-opens java.base/java.nio.charset=ALL-UNNAMED + --add-opens java.base/java.nio.file.attribute=ALL-UNNAMED + --add-opens java.base/java.lang.invoke=ALL-UNNAMED + --add-opens java.base/sun.net.www.protocol.jar=ALL-UNNAMED + --add-opens java.base/jdk.internal.perf=ALL-UNNAMED + + + ui diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index 60563704da132..48a7b1dd040e2 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -368,5 +368,26 @@ + + skipJdk17IncompatibleTests + + [17,) + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/TestAccumuloDistributedQueries.java + **/TestAccumuloIntegrationSmokeTest.java + **/TestAccumuloClient.java + + + + + + diff --git a/presto-base-arrow-flight/pom.xml b/presto-base-arrow-flight/pom.xml index a670a1a71a881..693f753ee0c0b 100644 --- a/presto-base-arrow-flight/pom.xml +++ b/presto-base-arrow-flight/pom.xml @@ -253,4 +253,18 @@ + + + + java17 + + 17 + + + + --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + + diff --git a/presto-druid/pom.xml b/presto-druid/pom.xml index 828be203be15a..c7e8f322a5056 100644 --- a/presto-druid/pom.xml +++ b/presto-druid/pom.xml @@ -130,6 +130,10 @@ it.unimi.dsi fastutil-core + + com.google.inject.extensions + guice-multibindings + diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index ddbf856f7a348..00629aaf8643b 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -630,13 +630,15 @@ test-hive-pushdown-filter-queries-basic + + 5g + org.apache.maven.plugins maven-surefire-plugin - -Xms4g -Xmx4g **/TestHivePushdownFilterQueries.java **/TestHivePushdownIntegrationSmokeTest.java diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 9c8d6dd1cb40c..7be0615d164f4 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -60,15 +60,27 @@ org.apache.kafka - kafka_2.12 + kafka-clients - org.apache.kafka - kafka-clients + org.apache.zookeeper + zookeeper + runtime + + 3.8.4 + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + - javax.inject javax.inject @@ -144,11 +156,6 @@ runtime - - org.apache.zookeeper - zookeeper - runtime - com.101tec @@ -174,6 +181,58 @@ test + + + org.apache.kafka + kafka_2.12 + ${dep.kafka.version} + test + test + + + commons-logging + commons-logging + + + + + + org.apache.kafka + kafka_2.12 + ${dep.kafka.version} + test + + + commons-logging + commons-logging + + + + + + org.apache.kafka + kafka-clients + ${dep.kafka.version} + test + test + + + + org.apache.kafka + kafka-server-common + ${dep.kafka.version} + test + test + + + + org.junit.jupiter + junit-jupiter + 5.10.2 + test + + + org.testng testng @@ -241,6 +300,13 @@ org.apache.maven.plugins maven-surefire-plugin + + + org.apache.maven.surefire + surefire-testng + 3.0.0-M7 + + @@ -248,6 +314,16 @@ + + org.basepom.maven + duplicate-finder-maven-plugin + + + kafka/kafka-version.properties + log4j.properties + + + diff --git a/presto-kafka/src/test/java/com/facebook/presto/kafka/TestMinimalFunctionality.java b/presto-kafka/src/test/java/com/facebook/presto/kafka/TestMinimalFunctionality.java index 7e8066e292b18..19505269101cb 100644 --- a/presto-kafka/src/test/java/com/facebook/presto/kafka/TestMinimalFunctionality.java +++ b/presto-kafka/src/test/java/com/facebook/presto/kafka/TestMinimalFunctionality.java @@ -90,8 +90,10 @@ public void spinUp() @AfterMethod(alwaysRun = true) public void tearDown() { - queryRunner.close(); - queryRunner = null; + if (queryRunner != null) { + queryRunner.close(); + queryRunner = null; + } } private void createMessages(String topicName, int count) diff --git a/presto-kafka/src/test/java/com/facebook/presto/kafka/util/EmbeddedKafka.java b/presto-kafka/src/test/java/com/facebook/presto/kafka/util/EmbeddedKafka.java index 8b0adfe9662f4..1dd3d1c1c397a 100644 --- a/presto-kafka/src/test/java/com/facebook/presto/kafka/util/EmbeddedKafka.java +++ b/presto-kafka/src/test/java/com/facebook/presto/kafka/util/EmbeddedKafka.java @@ -13,30 +13,28 @@ */ package com.facebook.presto.kafka.util; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; import com.google.common.io.Files; -import kafka.admin.AdminUtils; -import kafka.admin.RackAwareMode; -import kafka.server.KafkaConfig; -import kafka.server.KafkaServerStartable; -import kafka.utils.ZkUtils; +import kafka.testkit.KafkaClusterTestKit; +import kafka.testkit.TestKitNodes; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.common.serialization.LongSerializer; import java.io.Closeable; import java.io.File; import java.io.IOException; -import java.util.Map; +import java.util.Arrays; import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import static com.facebook.presto.kafka.util.TestUtils.findUnusedPort; -import static com.facebook.presto.kafka.util.TestUtils.toProperties; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; import static org.apache.kafka.clients.producer.ProducerConfig.ACKS_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.BOOTSTRAP_SERVERS_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; @@ -45,10 +43,9 @@ public class EmbeddedKafka implements Closeable { - private final EmbeddedZookeeper zookeeper; private final int port; private final File kafkaDataDir; - private final KafkaServerStartable kafka; + private final KafkaClusterTestKit kafka; private final AtomicBoolean started = new AtomicBoolean(); private final AtomicBoolean stopped = new AtomicBoolean(); @@ -56,50 +53,37 @@ public class EmbeddedKafka public static EmbeddedKafka createEmbeddedKafka() throws IOException { - return new EmbeddedKafka(new EmbeddedZookeeper(), new Properties()); + return new EmbeddedKafka(); } - public static EmbeddedKafka createEmbeddedKafka(Properties overrideProperties) + EmbeddedKafka() throws IOException { - return new EmbeddedKafka(new EmbeddedZookeeper(), overrideProperties); - } - - EmbeddedKafka(EmbeddedZookeeper zookeeper, Properties overrideProperties) - throws IOException - { - this.zookeeper = requireNonNull(zookeeper, "zookeeper is null"); - requireNonNull(overrideProperties, "overrideProperties is null"); - this.port = findUnusedPort(); this.kafkaDataDir = Files.createTempDir(); - - Map properties = ImmutableMap.builder() - .put("broker.id", "0") - .put("host.name", "localhost") - .put("num.partitions", "2") - .put("log.flush.interval.messages", "10000") - .put("log.flush.interval.ms", "1000") - .put("log.retention.minutes", "60") - .put("auto.create.topics.enable", "false") - .put("zookeeper.connection.timeout.ms", "1000000") - .put("port", Integer.toString(port)) - .put("log.dirs", kafkaDataDir.getAbsolutePath()) - .put("zookeeper.connect", zookeeper.getConnectString()) - .put("offsets.topic.replication.factor", "1") - .putAll(Maps.fromProperties(overrideProperties)) + TestKitNodes nodes = new TestKitNodes.Builder() + .setNumBrokerNodes(1) + .setNumControllerNodes(1) + .setCombined(true) .build(); - - KafkaConfig config = new KafkaConfig(toProperties(properties)); - this.kafka = new KafkaServerStartable(config); + try { + this.kafka = new KafkaClusterTestKit.Builder(nodes) + .setConfigProp("log.dirs", kafkaDataDir.getAbsolutePath()) + .build(); + } + catch (Exception e) { + throw new RuntimeException(e); + } } public void start() - throws InterruptedException, IOException + throws Exception { if (!started.getAndSet(true)) { - zookeeper.start(); + kafka.format(); kafka.startup(); + kafka.waitForActiveController(); + kafka.waitForReadyBrokers(); } } @@ -108,9 +92,12 @@ public void close() throws IOException { if (started.get() && !stopped.getAndSet(true)) { - kafka.shutdown(); - kafka.awaitShutdown(); - zookeeper.close(); + try { + kafka.close(); + } + catch (Exception e) { + throw new RuntimeException(e); + } deleteRecursively(kafkaDataDir.toPath(), ALLOW_INSECURE); } } @@ -124,14 +111,15 @@ public void createTopics(int partitions, int replication, Properties topicProper { checkState(started.get() && !stopped.get(), "not started!"); - ZkUtils zkUtils = ZkUtils.apply(getZookeeperConnectString(), 30_000, 30_000, false); - try { - for (String topic : topics) { - AdminUtils.createTopic(zkUtils, topic, partitions, replication, topicProperties, RackAwareMode.Disabled$.MODULE$); - } - } - finally { - zkUtils.close(); + try (AdminClient client = AdminClient.create(kafka.clientProperties())) { + client.createTopics(Arrays.stream(topics) + .map(topic -> new NewTopic(topic, partitions, (short) replication) + .configs(topicProperties.stringPropertyNames() + .stream() + .collect(toImmutableMap( + identity(), + prop -> (String) topicProperties.get(prop))))) + .collect(Collectors.toList())); } } @@ -146,23 +134,8 @@ public KafkaProducer createProducer() return new KafkaProducer<>(properties); } - public int getZookeeperPort() - { - return zookeeper.getPort(); - } - - public int getPort() - { - return port; - } - public String getConnectString() { - return "localhost:" + Integer.toString(port); - } - - public String getZookeeperConnectString() - { - return zookeeper.getConnectString(); + return kafka.bootstrapServers(); } } diff --git a/presto-main/etc/jvm.config b/presto-main/etc/jvm.config index 095e663b70c0f..311a1e242079d 100644 --- a/presto-main/etc/jvm.config +++ b/presto-main/etc/jvm.config @@ -4,3 +4,7 @@ # This configuration file is for development only and should NOT be used be # used in production. For example configuration, see the Presto documentation. # + +# Required for Java 17 runtime +#--add-opens=java.base/java.lang=ALL-UNNAMED +#--add-opens=java.base/java.lang.reflect=ALL-UNNAMED diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestKllSketchStateSerializer.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestKllSketchStateSerializer.java index be2e362ee4916..319d56bc94ca7 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestKllSketchStateSerializer.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestKllSketchStateSerializer.java @@ -25,6 +25,7 @@ import org.apache.datasketches.common.ArrayOfStringsSerDe; import org.apache.datasketches.kll.KllItemsSketch; import org.openjdk.jol.info.GraphLayout; +import org.testng.annotations.Ignore; import org.testng.annotations.Test; import java.util.Arrays; @@ -91,6 +92,7 @@ public void testEstimatedMemorySizeLong() testEstimatedMemorySize(BIGINT, i -> (long) i, .05); } + @Ignore("The memory size of a string-typed sketch is dependent upon the strings used as input, so this test doesn't verify much") @Test public void testEstimatedMemorySizeString() { diff --git a/presto-product-tests/conf/docker/common/compose-commons.sh b/presto-product-tests/conf/docker/common/compose-commons.sh index 5a87e5e1bc79a..eae9f18ce9583 100644 --- a/presto-product-tests/conf/docker/common/compose-commons.sh +++ b/presto-product-tests/conf/docker/common/compose-commons.sh @@ -1,6 +1,8 @@ # docker compose has a limited understanding of relative paths and interprets them relative to # compose-file location. We can't guarantee the shape of the paths coming from env variables, # so we canonicalize them. + +set -euo pipefail function export_canonical_path() { # make the path start with '/' or './'. Otherwise the result for 'foo.txt' is an absolute path. local PATH_REFERENCE=$1 @@ -26,6 +28,8 @@ source "${BASH_SOURCE%/*}/../../../bin/locations.sh" export DOCKER_IMAGES_VERSION=${DOCKER_IMAGES_VERSION:-10} export HADOOP_BASE_IMAGE=${HADOOP_BASE_IMAGE:-"prestodb/hdp2.6-hive"} +# This is the directory for the overriden JDK to use for starting Presto +export OVERRIDE_JDK_DIR=${OVERRIDE_JDK_DIR:-"/dev/null"} # The following variables are defined to enable running product tests with arbitrary/downloaded jars # and without building the project. The `presto.env` file should only be sourced if any of the variables diff --git a/presto-product-tests/conf/docker/common/kerberos.yml b/presto-product-tests/conf/docker/common/kerberos.yml index 8737347a9c3eb..144ae610a40e2 100644 --- a/presto-product-tests/conf/docker/common/kerberos.yml +++ b/presto-product-tests/conf/docker/common/kerberos.yml @@ -12,6 +12,8 @@ services: hostname: presto-master image: '${HADOOP_BASE_IMAGE}-kerberized:${DOCKER_IMAGES_VERSION}' command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh singlenode-kerberized run + volumes: + - ${OVERRIDE_JDK_DIR}:/docker/volumes/overridejdk networks: default: aliases: diff --git a/presto-product-tests/conf/docker/common/standard.yml b/presto-product-tests/conf/docker/common/standard.yml index 67a1993220fb9..c15ad89190fa3 100644 --- a/presto-product-tests/conf/docker/common/standard.yml +++ b/presto-product-tests/conf/docker/common/standard.yml @@ -39,6 +39,7 @@ services: volumes: - ${PRESTO_SERVER_DIR}:/docker/volumes/presto-server - ../../../src/main/resources/avro:/docker/volumes/presto-product-tests/avro + - ${OVERRIDE_JDK_DIR}:/docker/volumes/overridejdk application-runner: extends: diff --git a/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh b/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh index b804cba484611..b4a82d7d4ce1e 100755 --- a/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh +++ b/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -euo pipefail +set -euxo pipefail CONFIG="$1" @@ -14,9 +14,37 @@ if [[ ! -f "${CONFIG_PROPERTIES_LOCATION}" ]]; then exit 1 fi -/docker/volumes/presto-server/bin/launcher \ +JVM_CONFIG="${PRESTO_CONFIG_DIRECTORY}/jvm.config" +JAVA_PROPERTIES="" + +# If we have an overriden JDK volume mount, use it +# This is set to /dev/null ignore +if [ -d /docker/volumes/overridejdk ]; then + CONTAINER_JAVA_HOME=$JAVA_HOME + export JAVA_HOME=/docker/volumes/overridejdk + export PATH=$JAVA_HOME/bin:$PATH + + if [[ "$CONFIG" == "singlenode-ldap" ]]; then + # For LDAP tests use the cacert file from the container JDK which has certs installed + JAVA_PROPERTIES="-Djavax.net.ssl.trustStore=$CONTAINER_JAVA_HOME/jre/lib/security/cacerts" + fi +fi + +echo "Starting Presto with java set to :" +java -version + +# Check if Java version is 17 +# This relies on the version string adhering to the format "17.x.xx" +JAVA_VERSION=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | cut -d'.' -f1) +if [ "$JAVA_VERSION" == "17" ]; then + echo "Java version is 17, setting custom JVM config" + JVM_CONFIG="${PRESTO_CONFIG_DIRECTORY}/jvm17.config" +fi + +/docker/volumes/presto-server/bin/launcher \ -Dnode.id="${HOSTNAME}" \ --etc-dir="${PRESTO_CONFIG_DIRECTORY}" \ + --jvm-config="${JVM_CONFIG}" \ --config="${PRESTO_CONFIG_DIRECTORY}/${CONFIG}".properties \ - --data-dir=/var/presto \ + --data-dir=/var/presto ${JAVA_PROPERTIES} \ "$@" diff --git a/presto-product-tests/conf/docker/multinode-tls-kerberos/docker-compose.yml b/presto-product-tests/conf/docker/multinode-tls-kerberos/docker-compose.yml index 86e3f3ece9bd3..63e90275d47f3 100644 --- a/presto-product-tests/conf/docker/multinode-tls-kerberos/docker-compose.yml +++ b/presto-product-tests/conf/docker/multinode-tls-kerberos/docker-compose.yml @@ -13,6 +13,7 @@ services: - presto-master.docker.cluster volumes: - ../../../conf/presto/etc/environment-specific-catalogs/singlenode-kerberos-hdfs-no-impersonation/hive.properties:/docker/volumes/conf/presto/etc/catalog/hive.properties + - ${OVERRIDE_JDK_DIR}:/docker/volumes/overridejdk presto-worker-1: domainname: docker.cluster diff --git a/presto-product-tests/conf/docker/multinode-tls/docker-compose.yml b/presto-product-tests/conf/docker/multinode-tls/docker-compose.yml index 310feeb09e7f4..505be8a520d55 100644 --- a/presto-product-tests/conf/docker/multinode-tls/docker-compose.yml +++ b/presto-product-tests/conf/docker/multinode-tls/docker-compose.yml @@ -4,6 +4,8 @@ services: domainname: docker.cluster hostname: presto-master command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh multinode-tls-master run + volumes: + - ${OVERRIDE_JDK_DIR}:/docker/volumes/overridejdk ports: - '7778:7778' networks: diff --git a/presto-product-tests/conf/docker/multinode/docker-compose.yml b/presto-product-tests/conf/docker/multinode/docker-compose.yml index ef2b669b2242d..a7f7ebc2b3422 100644 --- a/presto-product-tests/conf/docker/multinode/docker-compose.yml +++ b/presto-product-tests/conf/docker/multinode/docker-compose.yml @@ -4,6 +4,8 @@ services: command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh multinode-master run volumes: - ../../../conf/presto/etc/multinode-master-jvm.config:/docker/volumes/conf/presto/etc/jvm.config + - ../../../conf/presto/etc/multinode-master-jvm17.config:/docker/volumes/conf/presto/etc/jvm17.config + - ${OVERRIDE_JDK_DIR}:/docker/volumes/overridejdk presto-worker: extends: @@ -16,3 +18,4 @@ services: - presto-master volumes: - ../../../conf/presto/etc/multinode-worker-jvm.config:/docker/volumes/conf/presto/etc/jvm.config + - ../../../conf/presto/etc/multinode-worker-jvm17.config:/docker/volumes/conf/presto/etc/jvm17.config diff --git a/presto-product-tests/conf/docker/singlenode-kafka/docker-compose.yml b/presto-product-tests/conf/docker/singlenode-kafka/docker-compose.yml index 0f50c2689fc62..766cc483ac858 100644 --- a/presto-product-tests/conf/docker/singlenode-kafka/docker-compose.yml +++ b/presto-product-tests/conf/docker/singlenode-kafka/docker-compose.yml @@ -2,12 +2,20 @@ services: kafka: hostname: kafka - image: spotify/kafka + image: apache/kafka:3.9.0 ports: - 9092:9092 - - 2181:2181 - command: | - bash -xeuo pipefail -c " - sed -i 's/#delete.topic.enable=true/delete.topic.enable=true/' /opt/kafka_2.11-0.10.1.0/config/server.properties - exec supervisord -n - " + environment: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://kafka:9092,CONTROLLER://kafka:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_NUM_PARTITIONS: 3 + KAFKA_DELETE_TOPIC_ENABLE: true diff --git a/presto-product-tests/conf/docker/singlenode-ldap/docker-compose.yml b/presto-product-tests/conf/docker/singlenode-ldap/docker-compose.yml index 03d15c000819f..b170d7d873dcb 100644 --- a/presto-product-tests/conf/docker/singlenode-ldap/docker-compose.yml +++ b/presto-product-tests/conf/docker/singlenode-ldap/docker-compose.yml @@ -9,6 +9,7 @@ services: - '8443:8443' volumes: - ../../../conf/presto/etc/ldap-authenticator.properties:/docker/volumes/conf/presto/etc/password-authenticator.properties + - ${OVERRIDE_JDK_DIR}:/docker/volumes/overridejdk application-runner: image: 'prestodb/centos6-oj8-openldap:${DOCKER_IMAGES_VERSION}' diff --git a/presto-product-tests/conf/presto/etc/catalog/mysql.properties b/presto-product-tests/conf/presto/etc/catalog/mysql.properties index bb6b444c91d09..475c1d6dc1146 100644 --- a/presto-product-tests/conf/presto/etc/catalog/mysql.properties +++ b/presto-product-tests/conf/presto/etc/catalog/mysql.properties @@ -1,4 +1,4 @@ connector.name=mysql -connection-url=jdbc:mysql://mysql:13306 +connection-url=jdbc:mysql://mysql:13306?enabledTLSProtocols=TLSv1.2 connection-user=root connection-password=swarm diff --git a/presto-product-tests/conf/presto/etc/jvm17.config b/presto-product-tests/conf/presto/etc/jvm17.config new file mode 100644 index 0000000000000..d1b67b63931bc --- /dev/null +++ b/presto-product-tests/conf/presto/etc/jvm17.config @@ -0,0 +1,40 @@ +# +# WARNING +# ^^^^^^^ +# This configuration file is for development only and should NOT be used be +# used in production. For example configuration, see the Presto documentation. +# + +-server +-Xmx2G +-XX:+UseG1GC +-XX:+UseGCOverheadLimit +-XX:+ExplicitGCInvokesConcurrent +-XX:+HeapDumpOnOutOfMemoryError +-XX:+ExitOnOutOfMemoryError +-XX:ReservedCodeCacheSize=150M +# jdk.nio.maxCachedBufferSize controls what buffers can be allocated in per-thread "temporary buffer cache" (sun.nio.ch.Util). Value of 0 disables the cache. +-Djdk.nio.maxCachedBufferSize=0 +-Djdk.attach.allowAttachSelf=true +-DHADOOP_USER_NAME=hive +-Duser.timezone=Asia/Kathmandu +#-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 +-XX:ErrorFile=/docker/volumes/logs/product-tests-presto-jvm-error-file.log +# We need to figure out which ones are actually needed, using the same ones as 'air.test.jvm.additional-arguments' +--add-opens=java.base/java.io=ALL-UNNAMED +--add-opens=java.base/java.lang=ALL-UNNAMED +--add-opens=java.base/java.lang.ref=ALL-UNNAMED +--add-opens=java.base/java.lang.reflect=ALL-UNNAMED +--add-opens=java.base/java.net=ALL-UNNAMED +--add-opens=java.base/java.nio=ALL-UNNAMED +--add-opens=java.base/java.security=ALL-UNNAMED +--add-opens=java.base/javax.security.auth=ALL-UNNAMED +--add-opens=java.base/javax.security.auth.login=ALL-UNNAMED +--add-opens=java.base/java.text=ALL-UNNAMED +--add-opens=java.base/java.util=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED +--add-opens=java.base/java.util.regex=ALL-UNNAMED +--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens=java.base/sun.security.action=ALL-UNNAMED +--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \ No newline at end of file diff --git a/presto-product-tests/conf/presto/etc/multinode-master-jvm17.config b/presto-product-tests/conf/presto/etc/multinode-master-jvm17.config new file mode 100644 index 0000000000000..521a7a58cc7e0 --- /dev/null +++ b/presto-product-tests/conf/presto/etc/multinode-master-jvm17.config @@ -0,0 +1,42 @@ +# +# WARNING +# ^^^^^^^ +# This configuration file is for development only and should NOT be used be +# used in production. For example configuration, see the Presto documentation. +# + +-server +# coordinator is not a worker +-Xmx1G +-XX:-UseBiasedLocking +# Disable G1 to enable UseGCOverheadLimit (JDK-8212084) and reduce JVM's memory footprint +-XX:+UseParallelGC +-XX:+ExitOnOutOfMemoryError +-XX:+UseGCOverheadLimit +-XX:+HeapDumpOnOutOfMemoryError +-XX:ReservedCodeCacheSize=150M +# jdk.nio.maxCachedBufferSize controls what buffers can be allocated in per-thread "temporary buffer cache" (sun.nio.ch.Util). Value of 0 disables the cache. +-Djdk.nio.maxCachedBufferSize=0 +-DHADOOP_USER_NAME=hive +-Duser.timezone=Asia/Kathmandu +#-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 +-XX:ErrorFile=/docker/volumes/logs/product-tests-presto-jvm-error-file.log +-Djdk.attach.allowAttachSelf=true +# We need to figure out which ones are actually needed, using the same ones as 'air.test.jvm.additional-arguments' +--add-opens=java.base/java.io=ALL-UNNAMED +--add-opens=java.base/java.lang=ALL-UNNAMED +--add-opens=java.base/java.lang.ref=ALL-UNNAMED +--add-opens=java.base/java.lang.reflect=ALL-UNNAMED +--add-opens=java.base/java.net=ALL-UNNAMED +--add-opens=java.base/java.nio=ALL-UNNAMED +--add-opens=java.base/java.security=ALL-UNNAMED +--add-opens=java.base/javax.security.auth=ALL-UNNAMED +--add-opens=java.base/javax.security.auth.login=ALL-UNNAMED +--add-opens=java.base/java.text=ALL-UNNAMED +--add-opens=java.base/java.util=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED +--add-opens=java.base/java.util.regex=ALL-UNNAMED +--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens=java.base/sun.security.action=ALL-UNNAMED +--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED diff --git a/presto-product-tests/conf/presto/etc/multinode-worker-jvm17.config b/presto-product-tests/conf/presto/etc/multinode-worker-jvm17.config new file mode 100644 index 0000000000000..71b7763ab5430 --- /dev/null +++ b/presto-product-tests/conf/presto/etc/multinode-worker-jvm17.config @@ -0,0 +1,42 @@ +# +# WARNING +# ^^^^^^^ +# This configuration file is for development only and should NOT be used +# in production. For example configuration, see the Presto documentation. +# + +-server +-Xmx2G +-XX:-UseBiasedLocking +# Disable G1 to enable UseGCOverheadLimit (JDK-8212084) and reduce JVM's memory footprint +-XX:+UseParallelGC +-XX:+ExitOnOutOfMemoryError +-XX:+UseGCOverheadLimit +-XX:+HeapDumpOnOutOfMemoryError +-XX:ReservedCodeCacheSize=150M +-Djdk.attach.allowAttachSelf=true +# jdk.nio.maxCachedBufferSize controls what buffers can be allocated in per-thread "temporary buffer cache" (sun.nio.ch.Util). Value of 0 disables the cache. +-Djdk.nio.maxCachedBufferSize=0 +-DHADOOP_USER_NAME=hive +-Duser.timezone=Asia/Kathmandu +#-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 +-XX:ErrorFile=/docker/volumes/logs/product-tests-presto-jvm-error-file.log +-Djdk.attach.allowAttachSelf=true +# We need to figure out which ones are actually needed, using the same ones as 'air.test.jvm.additional-arguments' +--add-opens=java.base/java.io=ALL-UNNAMED +--add-opens=java.base/java.lang=ALL-UNNAMED +--add-opens=java.base/java.lang.ref=ALL-UNNAMED +--add-opens=java.base/java.lang.reflect=ALL-UNNAMED +--add-opens=java.base/java.net=ALL-UNNAMED +--add-opens=java.base/java.nio=ALL-UNNAMED +--add-opens=java.base/java.security=ALL-UNNAMED +--add-opens=java.base/javax.security.auth=ALL-UNNAMED +--add-opens=java.base/javax.security.auth.login=ALL-UNNAMED +--add-opens=java.base/java.text=ALL-UNNAMED +--add-opens=java.base/java.util=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED +--add-opens=java.base/java.util.regex=ALL-UNNAMED +--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens=java.base/sun.security.action=ALL-UNNAMED +--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index cfd01c0c0f0b9..131eef0467c67 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -18,6 +18,50 @@ 2.12.2 + + + + org.apache.commons + commons-lang3 + 3.14.0 + + + + commons-cli + commons-cli + 1.4 + + + + + + + java17 + + 17 + + + + --add-opens java.base/java.io=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.lang.ref=ALL-UNNAMED + --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/java.net=ALL-UNNAMED + --add-opens java.base/java.nio=ALL-UNNAMED + --add-opens java.base/java.security=ALL-UNNAMED + --add-opens java.base/javax.security.auth=ALL-UNNAMED + --add-opens java.base/javax.security.auth.login=ALL-UNNAMED + --add-opens java.base/java.text=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.util.concurrent=ALL-UNNAMED + --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens java.base/java.util.regex=ALL-UNNAMED + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED + --add-opens java.base/sun.security.action=ALL-UNNAMED + + + + org.apache.avro @@ -136,11 +180,19 @@ com.datastax.cassandra cassandra-driver-core 3.11.5 + + shaded + io.netty * + + io.dropwizard.metrics + metrics-core + @@ -179,6 +231,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + ${project.build.targetJdk} + ${project.build.targetJdk} + + org.apache.maven.plugins maven-shade-plugin diff --git a/presto-product-tests/src/main/resources/tempto-configuration.yaml b/presto-product-tests/src/main/resources/tempto-configuration.yaml index 1c9762fec85b6..2b55eeadf5cc4 100644 --- a/presto-product-tests/src/main/resources/tempto-configuration.yaml +++ b/presto-product-tests/src/main/resources/tempto-configuration.yaml @@ -82,7 +82,7 @@ databases: mysql: jdbc_driver_class: com.mysql.jdbc.Driver - jdbc_url: jdbc:mysql://mysql:13306/test + jdbc_url: jdbc:mysql://mysql:13306/test?enabledTLSProtocols=TLSv1.2 jdbc_user: root jdbc_password: swarm jdbc_pooling: true diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 98a02babf58f5..f6e82e02b4cc0 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -13,6 +13,7 @@ ${project.parent.basedir} + 5g @@ -398,7 +399,6 @@ org.apache.maven.plugins maven-surefire-plugin - -Xms4g -Xmx4g **/TestDistributedQueriesNoHashGeneration.java