diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6c2f63dd49a6..c18fb09bdea9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -34,323 +34,14 @@ concurrency:
cancel-in-progress: true
jobs:
- maven-checks:
- runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- matrix:
- java-version:
- - 11
- - 17
- timeout-minutes: 45
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-java@v2
- with:
- distribution: 'zulu'
- java-version: ${{ matrix.java-version }}
- cache: 'maven'
- - name: Configure Problem Matchers
- run: |
- echo "::add-matcher::.github/problem-matcher.json"
- - name: Maven Checks
- run: |
- export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
- $RETRY $MAVEN install -B --strict-checksums -V -T C1 -DskipTests -P ci -pl '!:trino-server-rpm'
- - name: Test Server RPM
- run: |
- export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
- $RETRY bash -c "$MAVEN verify -B --strict-checksums -P ci -pl :trino-server-rpm || find core/trino-server-rpm/ -exec ls -ald {} +"
- - name: Clean Maven Output
- run: $MAVEN clean -pl '!:trino-server,!:trino-cli'
- - uses: docker/setup-qemu-action@v1
- with:
- platforms: arm64
- - name: Test Docker Image
- run: core/docker/build-local.sh
-
- error-prone-checks:
- runs-on: ubuntu-latest
- timeout-minutes: 45
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-java@v2
- with:
- distribution: 'zulu'
- java-version: 11
- cache: 'maven'
- - name: Configure Problem Matchers
- run: |
- echo "::add-matcher::.github/problem-matcher.json"
- - name: Maven Install
- run: |
- export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
- $RETRY $MAVEN install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server,!:trino-server-rpm'
- - name: Error Prone Checks
- run: |
- export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
- # Run Error Prone on one module with a retry to ensure all runtime dependencies are fetched
- $RETRY $MAVEN ${MAVEN_TEST} -T C1 clean test-compile -P errorprone-compiler -pl ':trino-spi'
- # The main Error Prone run
- $MAVEN ${MAVEN_TEST} -T C1 clean test-compile -P errorprone-compiler \
- -pl '!:trino-docs,!:trino-server,!:trino-server-rpm'
-
- web-ui-checks:
- runs-on: ubuntu-latest
- timeout-minutes: 30
- steps:
- - uses: actions/checkout@v2
- - name: Web UI Checks
- run: core/trino-main/bin/check_webui.sh
-
- test-jdbc-compatibility:
- runs-on: ubuntu-latest
- timeout-minutes: 30
- steps:
- - uses: actions/checkout@v2
- with:
- fetch-depth: 0 # checkout tags so version in Manifest is set properly
- - uses: actions/setup-java@v2
- with:
- distribution: 'zulu'
- java-version: 11
- cache: 'maven'
- - name: Configure Problem Matchers
- run: |
- echo "::add-matcher::.github/problem-matcher.json"
- - name: Maven Install
- run: |
- export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
- $RETRY $MAVEN install ${MAVEN_FAST_INSTALL} -pl '!:trino-test-jdbc-compatibility-old-driver,!:trino-docs,!:trino-server,!:trino-server-rpm'
- - name: Test old JDBC vs current server
- run: testing/trino-test-jdbc-compatibility-old-driver/bin/run_tests.sh
- - name: Test current JDBC vs old server
- if: always()
- run: $MAVEN test ${MAVEN_TEST} -pl :trino-test-jdbc-compatibility-old-server
- - name: Upload test results
- uses: actions/upload-artifact@v2
- # Upload all test reports only on failure, because the artifacts are large
- if: failure()
- with:
- name: result ${{ github.job }}
- path: |
- **/target/surefire-reports
- **/target/checkstyle-*
- - name: Upload test report
- uses: actions/upload-artifact@v2
- # Always upload the test report for the annotate.yml workflow,
- # but only the single XML file to keep the artifact small
- if: always()
- with:
- # Name prefix is checked in the `Annotate checks` workflow
- name: test report ${{ github.job }}
- path: |
- **/surefire-reports/TEST-*.xml
- retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
-
- hive-tests:
- runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- matrix:
- config:
- - config-empty
- - config-hdp3
- # TODO: config-cdh5
- # TODO: config-apache-hive3
- timeout-minutes: 60
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-java@v2
- with:
- distribution: 'zulu'
- java-version: 11
- cache: 'maven'
- - name: Configure Problem Matchers
- run: |
- echo "::add-matcher::.github/problem-matcher.json"
- - name: Install Hive Module
- run: |
- export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
- $RETRY $MAVEN install ${MAVEN_FAST_INSTALL} -am -pl :trino-hive-hadoop2
- - name: Run Hive Tests
- run: |
- source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
- plugin/trino-hive-hadoop2/bin/run_hive_tests.sh
- - name: Run Hive S3 Tests
- env:
- AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESSKEY }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETKEY }}
- S3_BUCKET: "presto-ci-test"
- S3_BUCKET_ENDPOINT: "s3.us-east-2.amazonaws.com"
- run: |
- if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then
- source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
- plugin/trino-hive-hadoop2/bin/run_hive_s3_tests.sh
- fi
- - name: Run Hive Glue Tests
- env:
- AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESSKEY }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETKEY }}
- AWS_REGION: us-east-2
- run: |
- if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then
- $MAVEN test ${MAVEN_TEST} -pl :trino-hive -P test-hive-glue
- fi
- - name: Run Hive Azure ABFS Access Key Tests
- if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
- env:
- ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
- ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
- ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }}
- run: |
- if [ "${ABFS_CONTAINER}" != "" ]; then
- source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
- plugin/trino-hive-hadoop2/bin/run_hive_abfs_access_key_tests.sh
- fi
- - name: Run Hive Azure ABFS OAuth Tests
- if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
- env:
- ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
- ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
- ABFS_OAUTH_ENDPOINT: ${{ secrets.AZURE_ABFS_OAUTH_ENDPOINT }}
- ABFS_OAUTH_CLIENTID: ${{ secrets.AZURE_ABFS_OAUTH_CLIENTID }}
- ABFS_OAUTH_SECRET: ${{ secrets.AZURE_ABFS_OAUTH_SECRET }}
- run: |
- if [ -n "$ABFS_CONTAINER" ]; then
- source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
- plugin/trino-hive-hadoop2/bin/run_hive_abfs_oauth_tests.sh
- fi
- - name: Run Hive Azure WASB Tests
- if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
- env:
- WASB_CONTAINER: ${{ secrets.AZURE_WASB_CONTAINER }}
- WASB_ACCOUNT: ${{ secrets.AZURE_WASB_ACCOUNT }}
- WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESSKEY }}
- run: |
- if [ "${WASB_CONTAINER}" != "" ]; then
- source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
- plugin/trino-hive-hadoop2/bin/run_hive_wasb_tests.sh
- fi
- - name: Run Hive Azure ADL Tests
- if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
- env:
- ADL_NAME: ${{ secrets.AZURE_ADL_NAME }}
- ADL_CLIENT_ID: ${{ secrets.AZURE_ADL_CLIENTID }}
- ADL_CREDENTIAL: ${{ secrets.AZURE_ADL_CREDENTIAL }}
- ADL_REFRESH_URL: ${{ secrets.AZURE_ADL_REFRESHURL }}
- run: |
- if [ "${ADL_NAME}" != "" ]; then
- source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
- plugin/trino-hive-hadoop2/bin/run_hive_adl_tests.sh
- fi
- - name: Run Hive Alluxio Tests
- run: |
- source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
- plugin/trino-hive-hadoop2/bin/run_hive_alluxio_tests.sh
- - name: Upload test results
- uses: actions/upload-artifact@v2
- # Upload all test reports only on failure, because the artifacts are large
- if: failure()
- with:
- name: result ${{ github.job }}
- path: |
- **/target/surefire-reports
- **/target/checkstyle-*
- - name: Upload test report
- uses: actions/upload-artifact@v2
- # Always upload the test report for the annotate.yml workflow,
- # but only the single XML file to keep the artifact small
- if: always()
- with:
- # Name prefix is checked in the `Annotate checks` workflow
- name: test report ${{ github.job }} (${{ matrix.config }})
- path: |
- **/surefire-reports/TEST-*.xml
- retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
-
- test-other-modules:
- runs-on: ubuntu-latest
- timeout-minutes: 60
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-java@v2
- with:
- distribution: 'zulu'
- java-version: 11
- cache: 'maven'
- - name: Configure Problem Matchers
- run: |
- echo "::add-matcher::.github/problem-matcher.json"
- - name: Maven Install
- run: |
- export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
- $RETRY $MAVEN install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server,!:trino-server-rpm'
- - name: Maven Tests
- run: |
- $MAVEN test ${MAVEN_TEST} -pl '
- !:trino-main,
- !:trino-tests,
- !:trino-raptor-legacy,
- !:trino-accumulo,
- !:trino-cassandra,
- !:trino-clickhouse,
- !:trino-hive,!:trino-orc,!:trino-parquet,
- !:trino-mongodb,!:trino-kafka,!:trino-elasticsearch,
- !:trino-redis,
- !:trino-sqlserver,!:trino-postgresql,!:trino-mysql,!:trino-memsql,
- !:trino-oracle,
- !:trino-kudu,
- !:trino-iceberg,!:trino-druid,
- !:trino-phoenix,!:trino-phoenix5,
- !:trino-docs,!:trino-server,!:trino-server-rpm,
- !:trino-test-jdbc-compatibility-old-server,
- !:trino-bigquery'
- - name: Upload test results
- uses: actions/upload-artifact@v2
- # Upload all test reports only on failure, because the artifacts are large
- if: failure()
- with:
- name: result ${{ github.job }}
- path: |
- **/target/surefire-reports
- **/target/checkstyle-*
- - name: Upload test report
- uses: actions/upload-artifact@v2
- # Always upload the test report for the annotate.yml workflow,
- # but only the single XML file to keep the artifact small
- if: always()
- with:
- # Name prefix is checked in the `Annotate checks` workflow
- name: test report ${{ github.job }}
- path: |
- **/surefire-reports/TEST-*.xml
- retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
-
test:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
modules:
- - ":trino-main"
- - ":trino-tests"
- - ":trino-raptor-legacy"
- - ":trino-accumulo"
- - ":trino-cassandra"
- ":trino-clickhouse"
- - ":trino-hive,:trino-orc"
- - ":trino-hive,:trino-parquet -P test-parquet"
- - ":trino-hive -P test-failure-recovery"
- - ":trino-hive -P test-fault-tolerant-execution"
- - ":trino-mongodb,:trino-kafka,:trino-elasticsearch"
- - ":trino-elasticsearch -P test-stats"
- - ":trino-redis"
- - ":trino-sqlserver,:trino-postgresql,:trino-mysql"
- - ":trino-oracle"
- - ":trino-kudu"
- - ":trino-iceberg,:trino-druid"
- - ":trino-phoenix,:trino-phoenix5"
+ i: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49]
timeout-minutes: 60
steps:
- uses: actions/checkout@v2
@@ -396,235 +87,3 @@ jobs:
path: |
**/surefire-reports/TEST-*.xml
retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
-
- test-memsql:
- runs-on: ubuntu-latest
- timeout-minutes: 30
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-java@v2
- with:
- distribution: 'zulu'
- java-version: 11
- cache: 'maven'
- - name: Configure Problem Matchers
- run: |
- echo "::add-matcher::.github/problem-matcher.json"
- - name: Cleanup node
- # This is required as a virtual environment update 20210219.1 left too little space for MemSQL to work
- run: .github/bin/cleanup-node.sh
- - name: Maven Install
- run: |
- export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
- $RETRY $MAVEN install ${MAVEN_FAST_INSTALL} -am -pl :trino-memsql
- - name: Memsql Tests
- env:
- MEMSQL_LICENSE: ${{ secrets.MEMSQL_LICENSE }}
- run: |
- if [ "${MEMSQL_LICENSE}" != "" ]; then
- $MAVEN test ${MAVEN_TEST} -pl :trino-memsql -Dmemsql.license=${MEMSQL_LICENSE}
- fi
- - name: Upload test results
- uses: actions/upload-artifact@v2
- # Upload all test reports only on failure, because the artifacts are large
- if: failure()
- with:
- name: result ${{ github.job }}
- path: |
- **/target/surefire-reports
- **/target/checkstyle-*
- - name: Upload test report
- uses: actions/upload-artifact@v2
- # Always upload the test report for the annotate.yml workflow,
- # but only the single XML file to keep the artifact small
- if: always()
- with:
- # Name prefix is checked in the `Annotate checks` workflow
- name: test report ${{ github.job }}
- path: |
- **/surefire-reports/TEST-*.xml
- retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
-
- test-bigquery:
- runs-on: ubuntu-latest
- timeout-minutes: 30
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-java@v2
- with:
- distribution: 'zulu'
- java-version: 11
- cache: 'maven'
- - name: Configure Problem Matchers
- run: |
- echo "::add-matcher::.github/problem-matcher.json"
- - name: Maven Install
- run: |
- export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
- $RETRY $MAVEN install ${MAVEN_FAST_INSTALL} -am -pl :trino-bigquery
- - name: Basic BigQuery Tests
- run: $MAVEN test ${MAVEN_TEST} -pl :trino-bigquery
- - name: Cloud BigQuery Tests
- env:
- BIGQUERY_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CREDENTIALS_KEY }}
- run: |
- if [ "${BIGQUERY_CREDENTIALS_KEY}" != "" ]; then
- $MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests -Dbigquery.credentials-key="${BIGQUERY_CREDENTIALS_KEY}"
- fi
- - name: Cloud BigQuery Case Insensitive Mapping Tests
- env:
- BIGQUERY_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY }}
- run: |
- if [ "${BIGQUERY_CREDENTIALS_KEY}" != "" ]; then
- $MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests-case-insensitive-mapping -Dbigquery.credentials-key="${BIGQUERY_CREDENTIALS_KEY}"
- fi
- - name: Upload test results
- uses: actions/upload-artifact@v2
- # Upload all test reports only on failure, because the artifacts are large
- if: failure()
- with:
- name: result ${{ github.job }}
- path: |
- **/target/surefire-reports
- **/target/checkstyle-*
- - name: Upload test report
- uses: actions/upload-artifact@v2
- # Always upload the test report for the annotate.yml workflow,
- # but only the single XML file to keep the artifact small
- if: always()
- with:
- # Name prefix is checked in the `Annotate checks` workflow
- name: test report ${{ github.job }}
- path: |
- **/surefire-reports/TEST-*.xml
- retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
-
- pt:
- runs-on: ubuntu-latest
- # explicitly define the name to avoid adding the value of the `ignore exclusion if` matrix item
- name: pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }})
- strategy:
- fail-fast: false
- matrix:
- config:
- - default
- - hdp3
- - cdh5
- # TODO: config-apache-hive3
- suite:
- - suite-1
- - suite-2
- - suite-3
- # suite-4 does not exist
- - suite-5
- jdk:
- - 11
- exclude:
- - config: cdh5
- ignore exclusion if: >-
- ${{ github.event_name != 'pull_request'
- || contains(github.event.pull_request.labels.*.name, 'tests:all')
- || contains(github.event.pull_request.labels.*.name, 'tests:hive')
- }}
-
- - config: default
- ignore exclusion if: >-
- ${{ github.event_name != 'pull_request'
- || contains(github.event.pull_request.labels.*.name, 'tests:all')
- || contains(github.event.pull_request.labels.*.name, 'tests:hive')
- }}
-
- ignore exclusion if:
- # Do not use this property outside of the matrix configuration.
- #
- # This is added to all matrix entries so they may be conditionally
- # excluded by adding them to the excludes list with a GHA expression
- # for this property.
- # - If the expression evaluates to true, it will never match the a
- # actual value of the property, and will therefore not be excluded.
- # - If the expression evaluates to false, it will match the actual
- # value of the property, and the exclusion will apply normally.
- - false
- include:
- # this suite is not meant to be run with different configs
- - config: default
- suite: suite-6-non-generic
- jdk: 11
- # this suite is not meant to be run with different configs
- - config: default
- suite: suite-7-non-generic
- jdk: 11
- # this suite is not meant to be run with different configs
- - config: default
- suite: suite-8-non-generic
- jdk: 11
- # this suite is not meant to be run with different configs
- - config: default
- suite: suite-tpcds
- jdk: 11
- # this suite is not meant to be run with different configs
- - config: default
- suite: suite-oauth2
- jdk: 11
- # this suite is not meant to be run with different configs
- - config: default
- suite: suite-ldap
- jdk: 11
- # this suite is not meant to be run with different configs
- - config: default
- suite: suite-compatibility
- jdk: 11
- # this suite is designed specifically for apache-hive3. TODO remove the suite once we can run all regular tests on apache-hive3.
- - config: apache-hive3
- suite: suite-hms-only
- jdk: 11
- # suite-1 does not contain Kerberos/encrypted Hive environments
- - config: hdp3
- suite: suite-1
- jdk: 17
- # suite-2 contains Kerberos/encrypted Hive environments
- - config: hdp3
- suite: suite-2
- jdk: 17
- # PT Launcher's timeout defaults to 2h, account for preparation steps (compilation) and add some margin
- timeout-minutes: 140
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-java@v2
- with:
- distribution: 'zulu'
- java-version: 11
- cache: 'maven'
- - name: Maven Install
- run: |
- export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
- $RETRY $MAVEN install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server-rpm'
- - name: Free Disk Space
- run: |
- docker image prune -af
- sudo apt-get clean
- rm -rf ~/.m2/repository
- - name: Product Tests
- run: |
- testing/bin/ptl suite run \
- --suite ${{ matrix.suite }} --config config-${{ matrix.config }} --bind=off --logs-dir logs/ --timeout 2h \
- --trino-jdk-version zulu_${{ matrix.jdk }}
- - name: Upload test logs and results
- uses: actions/upload-artifact@v2
- # Upload all test reports only on failure, because the artifacts are large
- if: failure()
- with:
- name: result pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }})
- path: |
- testing/trino-product-tests/target/*
- logs/*
- - name: Upload test report
- uses: actions/upload-artifact@v2
- # Always upload the test report for the annotate.yml workflow,
- # but only the single XML file to keep the artifact small
- if: always()
- with:
- # Name prefix is checked in the `Annotate checks` workflow
- name: test report pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }})
- path: testing/trino-product-tests/target/reports/**/testng-results.xml
- retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
diff --git a/plugin/trino-clickhouse/pom.xml b/plugin/trino-clickhouse/pom.xml
index bef759aa5d55..c403773a3b56 100644
--- a/plugin/trino-clickhouse/pom.xml
+++ b/plugin/trino-clickhouse/pom.xml
@@ -46,27 +46,12 @@
com.clickhouse
clickhouse-jdbc
- 0.3.2-patch1
+ 0.3.2-patch3
+ all
- org.slf4j
- slf4j-api
-
-
- commons-logging
- commons-logging
-
-
- org.projectlombok
- lombok
-
-
- com.clickhouse
- clickhouse-http-client
-
-
- org.robolectric
- android-all
+ *
+ *
@@ -202,4 +187,21 @@
test
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+
+
+ org.apache.maven.surefire
+ surefire-testng
+ ${dep.plugin.surefire.version}
+
+
+
+
+
diff --git a/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClient.java b/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClient.java
index 63f1817a5477..7fb057826874 100644
--- a/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClient.java
+++ b/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClient.java
@@ -13,9 +13,12 @@
*/
package io.trino.plugin.clickhouse;
+import com.clickhouse.client.ClickHouseColumn;
+import com.clickhouse.client.ClickHouseDataType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.net.InetAddresses;
+import io.airlift.log.Logger;
import io.airlift.slice.Slice;
import io.trino.plugin.base.expression.AggregateFunctionRewriter;
import io.trino.plugin.base.expression.AggregateFunctionRule;
@@ -61,10 +64,10 @@
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.sql.Connection;
-import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Statement;
import java.sql.Types;
import java.time.LocalDate;
import java.time.LocalDateTime;
@@ -142,6 +145,8 @@
public class ClickHouseClient
extends BaseJdbcClient
{
+ private static final Logger log = Logger.get(ClickHouseClient.class);
+
static final int CLICKHOUSE_MAX_DECIMAL_PRECISION = 76;
private static final long MIN_SUPPORTED_DATE_EPOCH = LocalDate.parse("1970-01-01").toEpochDay();
private static final long MAX_SUPPORTED_DATE_EPOCH = LocalDate.parse("2106-02-07").toEpochDay(); // The max date is '2148-12-31' in new ClickHouse version
@@ -342,16 +347,9 @@ public void setColumnComment(ConnectorSession session, JdbcTableHandle handle, J
}
@Override
- public ResultSet getTables(Connection connection, Optional schemaName, Optional tableName)
- throws SQLException
+ protected Optional> getTableTypes()
{
- // ClickHouse maps their "database" to SQL catalogs and does not have schemas
- DatabaseMetaData metadata = connection.getMetaData();
- return metadata.getTables(
- null,
- schemaName.orElse(null),
- escapeNamePattern(tableName, metadata.getSearchStringEscape()).orElse(null),
- new String[] {"TABLE", "VIEW"});
+ return Optional.empty();
}
@Override
@@ -402,13 +400,15 @@ public Optional toColumnMapping(ConnectorSession session, Connect
return mapping;
}
- switch (jdbcTypeName.replaceAll("\\(.*\\)$", "")) {
- case "IPv4":
+ ClickHouseColumn column = ClickHouseColumn.of("", jdbcTypeName);
+ ClickHouseDataType columnDataType = column.getDataType();
+ switch (columnDataType) {
+ case IPv4:
return Optional.of(ipAddressColumnMapping("IPv4StringToNum(?)"));
- case "IPv6":
+ case IPv6:
return Optional.of(ipAddressColumnMapping("IPv6StringToNum(?)"));
- case "Enum8":
- case "Enum16":
+ case Enum8:
+ case Enum16:
return Optional.of(ColumnMapping.sliceMapping(
createUnboundedVarcharType(),
varcharReadFunction(createUnboundedVarcharType()),
@@ -416,8 +416,8 @@ public Optional toColumnMapping(ConnectorSession session, Connect
// TODO (https://github.com/trinodb/trino/issues/7100) Currently pushdown would not work and may require a custom bind expression
DISABLE_PUSHDOWN));
- case "FixedString": // FixedString(n)
- case "String":
+ case FixedString: // FixedString(n)
+ case String:
if (isMapStringAsVarchar(session)) {
return Optional.of(ColumnMapping.sliceMapping(
createUnboundedVarcharType(),
@@ -427,7 +427,7 @@ public Optional toColumnMapping(ConnectorSession session, Connect
}
// TODO (https://github.com/trinodb/trino/issues/7100) test & enable predicate pushdown
return Optional.of(varbinaryColumnMapping());
- case "UUID":
+ case UUID:
return Optional.of(uuidColumnMapping());
}
@@ -478,7 +478,7 @@ public Optional toColumnMapping(ConnectorSession session, Connect
return Optional.of(dateColumnMappingUsingLocalDate());
case Types.TIMESTAMP:
- if (jdbcTypeName.equals("DateTime")) {
+ if (columnDataType == ClickHouseDataType.DateTime) {
verify(typeHandle.getRequiredDecimalDigits() == 0, "Expected 0 as timestamp precision, but got %s", typeHandle.getRequiredDecimalDigits());
return Optional.of(ColumnMapping.longMapping(
TIMESTAMP_SECONDS,
@@ -546,6 +546,27 @@ public WriteMapping toWriteMapping(ConnectorSession session, Type type)
throw new TrinoException(NOT_SUPPORTED, "Unsupported column type: " + type);
}
+ @Override
+ protected void execute(Connection connection, String query)
+ {
+ debug(connection);
+ super.execute(connection, query);
+ debug(connection);
+ }
+
+ private void debug(Connection connection)
+ {
+ try (Statement statement = connection.createStatement()) {
+ ResultSet resultSet = statement.executeQuery("SELECT version()");
+ while (resultSet.next()) {
+ log.info("Server version: %s", resultSet.getString(1));
+ }
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Failed to execute statement", e);
+ }
+ }
+
/**
* format property to match ClickHouse create table statement
*
diff --git a/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClientModule.java b/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClientModule.java
index 1a538cda6dd2..2e58814d8c18 100644
--- a/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClientModule.java
+++ b/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClientModule.java
@@ -15,9 +15,7 @@
import com.google.inject.Binder;
import com.google.inject.Module;
-import com.google.inject.Provides;
import com.google.inject.Scopes;
-import com.google.inject.Singleton;
import io.airlift.configuration.ConfigBinder;
import io.trino.plugin.jdbc.BaseJdbcConfig;
import io.trino.plugin.jdbc.ConnectionFactory;
@@ -26,14 +24,23 @@
import io.trino.plugin.jdbc.ForBaseJdbc;
import io.trino.plugin.jdbc.JdbcClient;
import io.trino.plugin.jdbc.credential.CredentialProvider;
-import ru.yandex.clickhouse.ClickHouseDriver;
+
+import javax.inject.Inject;
+import javax.inject.Provider;
+
+import java.sql.Driver;
import static io.trino.plugin.jdbc.JdbcModule.bindSessionPropertiesProvider;
import static io.trino.plugin.jdbc.JdbcModule.bindTablePropertiesProvider;
+import static java.lang.String.format;
public class ClickHouseClientModule
implements Module
{
+ private static final String CLICKHOUSE_LATEST_DRIVER_CLASS_NAME = "com.clickhouse.jdbc.ClickHouseDriver";
+ // TODO: This Driver will not be available when clickhouse-jdbc is upgraded to 0.4.0 or above
+ private static final String CLICKHOUSE_DEPRECATED_DRIVER_CLASS_NAME = "ru.yandex.clickhouse.ClickHouseDriver";
+
@Override
public void configure(Binder binder)
{
@@ -42,13 +49,53 @@ public void configure(Binder binder)
binder.bind(JdbcClient.class).annotatedWith(ForBaseJdbc.class).to(ClickHouseClient.class).in(Scopes.SINGLETON);
bindTablePropertiesProvider(binder, ClickHouseTableProperties.class);
binder.install(new DecimalModule());
+ binder.bind(ConnectionFactory.class).annotatedWith(ForBaseJdbc.class).toProvider(ConnectionFactoryProvider.class).in(Scopes.SINGLETON);
}
- @Provides
- @Singleton
- @ForBaseJdbc
- public static ConnectionFactory createConnectionFactory(BaseJdbcConfig config, CredentialProvider credentialProvider)
+ private static class ConnectionFactoryProvider
+ implements Provider
{
- return new ClickHouseConnectionFactory(new DriverConnectionFactory(new ClickHouseDriver(), config, credentialProvider));
+ private final ConnectionFactory connectionFactory;
+
+ @Inject
+ public ConnectionFactoryProvider(ClickHouseConfig clickHouseConfig, BaseJdbcConfig baseJdbcConfig, CredentialProvider credentialProvider)
+ {
+ connectionFactory = new ClickHouseConnectionFactory(
+ new DriverConnectionFactory(createDriver(clickHouseConfig.isUseDeprecatedDriver()), baseJdbcConfig, credentialProvider));
+ }
+
+ @Override
+ public ConnectionFactory get()
+ {
+ return connectionFactory;
+ }
+
+ Driver createDriver(boolean useDeprecatedDriver)
+ {
+ String driverClass = getDriverClassName(useDeprecatedDriver);
+ try {
+ return (Driver) Class.forName(driverClass, true, getClassLoader()).getDeclaredConstructor().newInstance();
+ }
+ catch (ReflectiveOperationException e) {
+ throw new RuntimeException(format("Error creating an instance of %s", driverClass), e);
+ }
+ }
+
+ String getDriverClassName(boolean useDeprecatedDriver)
+ {
+ if (useDeprecatedDriver) {
+ return CLICKHOUSE_DEPRECATED_DRIVER_CLASS_NAME;
+ }
+ return CLICKHOUSE_LATEST_DRIVER_CLASS_NAME;
+ }
+
+ ClassLoader getClassLoader()
+ {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = ClickHouseConfig.class.getClassLoader();
+ }
+ return classLoader;
+ }
}
}
diff --git a/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseConfig.java b/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseConfig.java
index f1cb97d7de39..e966e834b85e 100644
--- a/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseConfig.java
+++ b/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseConfig.java
@@ -21,6 +21,9 @@ public class ClickHouseConfig
// TODO (https://github.com/trinodb/trino/issues/7102) reconsider default behavior
private boolean mapStringAsVarchar;
+ // TODO: This config needs to be deprecated when we upgrade clickhouse-jdbc to version 0.4.0 or above
+ private boolean useDeprecatedDriver = true;
+
public boolean isMapStringAsVarchar()
{
return mapStringAsVarchar;
@@ -33,4 +36,17 @@ public ClickHouseConfig setMapStringAsVarchar(boolean mapStringAsVarchar)
this.mapStringAsVarchar = mapStringAsVarchar;
return this;
}
+
+ public boolean isUseDeprecatedDriver()
+ {
+ return useDeprecatedDriver;
+ }
+
+ @Config("clickhouse.use-deprecated-driver")
+ @ConfigDescription("Whether to use a deprecated driver")
+ public ClickHouseConfig setUseDeprecatedDriver(boolean useDeprecatedDriver)
+ {
+ this.useDeprecatedDriver = useDeprecatedDriver;
+ return this;
+ }
}
diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/BaseClickHouseConnectorSmokeTest.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/BaseClickHouseConnectorSmokeTest.java
index 46a2fceadff1..d6ac0d86b8e3 100644
--- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/BaseClickHouseConnectorSmokeTest.java
+++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/BaseClickHouseConnectorSmokeTest.java
@@ -15,7 +15,6 @@
import io.trino.plugin.jdbc.BaseJdbcConnectorSmokeTest;
import io.trino.testing.TestingConnectorBehavior;
-import org.testng.annotations.Test;
public abstract class BaseClickHouseConnectorSmokeTest
extends BaseJdbcConnectorSmokeTest
@@ -31,8 +30,6 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior)
}
}
- // TODO (https://github.com/trinodb/trino/issues/10653) Disable until fixing the flaky test issue
- @Test(enabled = false)
@Override
public void testRenameSchema()
{
diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/ClickHouseQueryRunner.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/ClickHouseQueryRunner.java
index 5338679e0414..d1394c67af14 100644
--- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/ClickHouseQueryRunner.java
+++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/ClickHouseQueryRunner.java
@@ -88,10 +88,11 @@ public static void main(String[] args)
{
Logging.initialize();
+ TestingClickHouseServer clickHouseServer = new TestingClickHouseServer();
DistributedQueryRunner queryRunner = createClickHouseQueryRunner(
- new TestingClickHouseServer(),
+ clickHouseServer,
ImmutableMap.of("http-server.http.port", "8080"),
- ImmutableMap.of(),
+ ImmutableMap.of("clickhouse.use-deprecated-driver", String.valueOf(!clickHouseServer.isLatestDriverMinimumSupportedVersion())),
TpchTable.getTables());
Logger log = Logger.get(ClickHouseQueryRunner.class);
diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestAltinityConnectorSmokeTest.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestAltinityConnectorSmokeTest.java
index ab9356545922..d12a1ffb79a5 100644
--- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestAltinityConnectorSmokeTest.java
+++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestAltinityConnectorSmokeTest.java
@@ -14,7 +14,9 @@
package io.trino.plugin.clickhouse;
import com.google.common.collect.ImmutableMap;
+import io.airlift.log.Logger;
import io.trino.testing.QueryRunner;
+import org.testng.annotations.Test;
import static io.trino.plugin.clickhouse.ClickHouseQueryRunner.createClickHouseQueryRunner;
import static io.trino.plugin.clickhouse.TestingClickHouseServer.ALTINITY_DEFAULT_IMAGE;
@@ -23,19 +25,26 @@
public class TestAltinityConnectorSmokeTest
extends BaseClickHouseConnectorSmokeTest
{
+ private static final Logger log = Logger.get(TestAltinityConnectorSmokeTest.class);
+
+ private TestingClickHouseServer clickHouseServer;
+
@Override
protected QueryRunner createQueryRunner()
throws Exception
{
+ clickHouseServer = closeAfterClass(new TestingClickHouseServer(ALTINITY_DEFAULT_IMAGE));
return createClickHouseQueryRunner(
- closeAfterClass(new TestingClickHouseServer(ALTINITY_DEFAULT_IMAGE)),
+ clickHouseServer,
ImmutableMap.of(),
ImmutableMap.builder()
.put("clickhouse.map-string-as-varchar", "true") // To handle string types in TPCH tables as varchar instead of varbinary
+ .put("clickhouse.use-deprecated-driver", String.valueOf(!clickHouseServer.isLatestDriverMinimumSupportedVersion()))
.buildOrThrow(),
REQUIRED_TPCH_TABLES);
}
+ @Test
@Override
public void testRenameSchema()
{
diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConfig.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConfig.java
index ce9159188aef..d7f0cab05f39 100644
--- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConfig.java
+++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConfig.java
@@ -42,7 +42,8 @@ public class TestClickHouseConfig
public void testDefaults()
{
assertRecordedDefaults(recordDefaults(ClickHouseConfig.class)
- .setMapStringAsVarchar(false));
+ .setMapStringAsVarchar(false)
+ .setUseDeprecatedDriver(true));
}
@Test
@@ -50,10 +51,12 @@ public void testExplicitPropertyMappings()
{
Map properties = new ImmutableMap.Builder()
.put("clickhouse.map-string-as-varchar", "true")
+ .put("clickhouse.use-deprecated-driver", "false")
.buildOrThrow();
ClickHouseConfig expected = new ClickHouseConfig()
- .setMapStringAsVarchar(true);
+ .setMapStringAsVarchar(true)
+ .setUseDeprecatedDriver(false);
assertFullMapping(properties, expected);
}
diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorSmokeTest.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorSmokeTest.java
index a7a53239ca35..5d6a60c97796 100644
--- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorSmokeTest.java
+++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorSmokeTest.java
@@ -33,6 +33,7 @@ protected QueryRunner createQueryRunner()
ImmutableMap.of(),
ImmutableMap.builder()
.put("clickhouse.map-string-as-varchar", "true") // To handle string types in TPCH tables as varchar instead of varbinary
+ .put("clickhouse.use-deprecated-driver", String.valueOf(!clickHouseServer.isLatestDriverMinimumSupportedVersion()))
.buildOrThrow(),
REQUIRED_TPCH_TABLES);
}
diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorTest.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorTest.java
index db5fc48fd677..5e2457a8f1fa 100644
--- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorTest.java
+++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorTest.java
@@ -65,6 +65,7 @@ protected QueryRunner createQueryRunner()
ImmutableMap.of(),
ImmutableMap.builder()
.put("clickhouse.map-string-as-varchar", "true")
+ .put("clickhouse.use-deprecated-driver", String.valueOf(!clickhouseServer.isLatestDriverMinimumSupportedVersion()))
.buildOrThrow(),
REQUIRED_TPCH_TABLES);
}
@@ -129,8 +130,10 @@ public void testDropColumn()
// the columns are referenced by order_by/order_by property can not be dropped
assertUpdate("CREATE TABLE " + tableName + "(x int NOT NULL, y int, a int NOT NULL) WITH " +
"(engine = 'MergeTree', order_by = ARRAY['x'], partition_by = ARRAY['a'])");
- assertQueryFails("ALTER TABLE " + tableName + " DROP COLUMN x", "ClickHouse exception, code: 47,.*\\n");
- assertQueryFails("ALTER TABLE " + tableName + " DROP COLUMN a", "ClickHouse exception, code: 47,.*\\n");
+ assertQueryFails("ALTER TABLE " + tableName + " DROP COLUMN x",
+ "Code: 47,.* Missing columns: 'x' while processing query: 'x', required columns: 'x' 'x' .*\\n.*");
+ assertQueryFails("ALTER TABLE " + tableName + " DROP COLUMN a",
+ "Code: 47,.* Missing columns: 'a' while processing query: 'a', required columns: 'a' 'a' .*\\n.*");
}
@Override
@@ -292,9 +295,12 @@ public void testTableProperty()
assertUpdate("DROP TABLE " + tableName);
// Log engine DOES NOT any property
- assertQueryFails("CREATE TABLE " + tableName + " (id int NOT NULL, x VARCHAR) WITH (engine = 'Log', order_by=ARRAY['id'])", ".* doesn't support PARTITION_BY, PRIMARY_KEY, ORDER_BY or SAMPLE_BY clauses.*\\n");
- assertQueryFails("CREATE TABLE " + tableName + " (id int NOT NULL, x VARCHAR) WITH (engine = 'Log', partition_by=ARRAY['id'])", ".* doesn't support PARTITION_BY, PRIMARY_KEY, ORDER_BY or SAMPLE_BY clauses.*\\n");
- assertQueryFails("CREATE TABLE " + tableName + " (id int NOT NULL, x VARCHAR) WITH (engine = 'Log', sample_by='id')", ".* doesn't support PARTITION_BY, PRIMARY_KEY, ORDER_BY or SAMPLE_BY clauses.*\\n");
+ assertQueryFails("CREATE TABLE " + tableName + " (id int NOT NULL, x VARCHAR) WITH (engine = 'Log', order_by=ARRAY['id'])",
+ ".* doesn't support PARTITION_BY, PRIMARY_KEY, ORDER_BY or SAMPLE_BY clauses.*\\n.*");
+ assertQueryFails("CREATE TABLE " + tableName + " (id int NOT NULL, x VARCHAR) WITH (engine = 'Log', partition_by=ARRAY['id'])",
+ ".* doesn't support PARTITION_BY, PRIMARY_KEY, ORDER_BY or SAMPLE_BY clauses.*\\n.*");
+ assertQueryFails("CREATE TABLE " + tableName + " (id int NOT NULL, x VARCHAR) WITH (engine = 'Log', sample_by='id')",
+ ".* doesn't support PARTITION_BY, PRIMARY_KEY, ORDER_BY or SAMPLE_BY clauses.*\\n.*");
// optional properties
assertUpdate("CREATE TABLE " + tableName + " (id int NOT NULL, x VARCHAR) WITH (engine = 'MergeTree', order_by = ARRAY['id'])");
@@ -302,7 +308,8 @@ public void testTableProperty()
assertUpdate("DROP TABLE " + tableName);
// the column refers by order by must be not null
- assertQueryFails("CREATE TABLE " + tableName + " (id int NOT NULL, x VARCHAR) WITH (engine = 'MergeTree', order_by = ARRAY['id', 'x'])", ".* Sorting key cannot contain nullable columns.*\\n");
+ assertQueryFails("CREATE TABLE " + tableName + " (id int NOT NULL, x VARCHAR) WITH (engine = 'MergeTree', order_by = ARRAY['id', 'x'])",
+ ".* Sorting key cannot contain nullable columns.*\\n.*");
assertUpdate("CREATE TABLE " + tableName + " (id int NOT NULL, x VARCHAR) WITH (engine = 'MergeTree', order_by = ARRAY['id'], primary_key = ARRAY['id'])");
assertTrue(getQueryRunner().tableExists(getSession(), tableName));
diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseTypeMapping.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseTypeMapping.java
index f96b84e1504a..fd8ef7116ffa 100644
--- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseTypeMapping.java
+++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseTypeMapping.java
@@ -28,7 +28,6 @@
import io.trino.testing.datatype.SqlDataTypeTest;
import io.trino.testing.sql.TestTable;
import io.trino.testing.sql.TrinoSqlExecutor;
-import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@@ -106,21 +105,16 @@ private static void checkIsDoubled(ZoneId zone, LocalDateTime dateTime)
protected QueryRunner createQueryRunner()
throws Exception
{
- clickhouseServer = new TestingClickHouseServer();
+ clickhouseServer = closeAfterClass(new TestingClickHouseServer());
return createClickHouseQueryRunner(clickhouseServer, ImmutableMap.of(),
ImmutableMap.builder()
.put("metadata.cache-ttl", "10m")
.put("metadata.cache-missing", "true")
+ .put("clickhouse.use-deprecated-driver", String.valueOf(!clickhouseServer.isLatestDriverMinimumSupportedVersion()))
.buildOrThrow(),
ImmutableList.of());
}
- @AfterClass(alwaysRun = true)
- public final void destroy()
- {
- clickhouseServer.close();
- }
-
@Test
public void testBasicTypes()
{
diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestingClickHouseServer.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestingClickHouseServer.java
index b4b053ae8c7d..d2cfccdb1446 100644
--- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestingClickHouseServer.java
+++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestingClickHouseServer.java
@@ -13,31 +13,49 @@
*/
package io.trino.plugin.clickhouse;
+import com.clickhouse.client.ClickHouseVersion;
+import io.airlift.log.Logger;
import org.testcontainers.containers.ClickHouseContainer;
import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.MountableFile;
import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.SecureRandom;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import static java.lang.String.format;
-import static org.testcontainers.containers.ClickHouseContainer.HTTP_PORT;
-import static org.testcontainers.utility.MountableFile.forClasspathResource;
public class TestingClickHouseServer
+ extends ClickHouseContainer
implements Closeable
{
+ private static final Logger log = Logger.get(TestingClickHouseServer.class);
+
private static final DockerImageName CLICKHOUSE_IMAGE = DockerImageName.parse("yandex/clickhouse-server");
public static final DockerImageName CLICKHOUSE_LATEST_IMAGE = CLICKHOUSE_IMAGE.withTag("21.11.10.1");
public static final DockerImageName CLICKHOUSE_DEFAULT_IMAGE = CLICKHOUSE_IMAGE.withTag("21.3.2.5"); // EOL is 30 Mar 2022
+ private static final String CLICKHOUSE_LATEST_DRIVER_CLASS_NAME = "com.clickhouse.jdbc.ClickHouseDriver";
+ // TODO: This Driver will not be available when clickhouse-jdbc is upgraded to 0.4.0 or above
+ private static final String CLICKHOUSE_DEPRECATED_DRIVER_CLASS_NAME = "ru.yandex.clickhouse.ClickHouseDriver";
+ private static final String CLICKHOUSE_LATEST_DRIVER_MINIMUM_SUPPORTED_VERSION = "20.7";
+
// Altinity Stable Builds Life-Cycle Table https://docs.altinity.com/altinitystablebuilds/#altinity-stable-builds-life-cycle-table
private static final DockerImageName ALTINITY_IMAGE = DockerImageName.parse("altinity/clickhouse-server").asCompatibleSubstituteFor("yandex/clickhouse-server");
- public static final DockerImageName ALTINITY_LATEST_IMAGE = ALTINITY_IMAGE.withTag("21.8.13.1.altinitystable");
+ public static final DockerImageName ALTINITY_LATEST_IMAGE = ALTINITY_IMAGE.withTag("21.8.12.29.altinitystable");
public static final DockerImageName ALTINITY_DEFAULT_IMAGE = ALTINITY_IMAGE.withTag("20.3.12-aes"); // EOL is 24 June 2022
- private final ClickHouseContainer dockerContainer;
+ private static final SecureRandom RANDOM = new SecureRandom();
+
+ private final int port;
+ private final boolean latestDriverMinimumSupportedVersion;
public TestingClickHouseServer()
{
@@ -46,11 +64,38 @@ public TestingClickHouseServer()
public TestingClickHouseServer(DockerImageName image)
{
- dockerContainer = (ClickHouseContainer) new ClickHouseContainer(image)
- .withCopyFileToContainer(forClasspathResource("custom.xml"), "/etc/clickhouse-server/config.d/custom.xml")
- .withStartupAttempts(10);
+ super(image);
+ port = 40000 + RANDOM.nextInt(1000);
+ latestDriverMinimumSupportedVersion = ClickHouseVersion.of(image.getVersionPart()).isNewerOrEqualTo(CLICKHOUSE_LATEST_DRIVER_MINIMUM_SUPPORTED_VERSION);
+ log.info("image: %s, port: %s", image, port);
+ withCopyFileToContainer(MountableFile.forHostPath(generateConfig(port)), "/etc/clickhouse-server/config.d/custom.xml");
+ withExposedPorts(port);
+ withStartupAttempts(10);
+ start();
+ }
- dockerContainer.start();
+ private static Path generateConfig(int port)
+ {
+ try {
+ File tempFile = File.createTempFile("custom-", ".xml");
+ String string = format("" +
+ "\n" +
+ "\n" +
+ " \n" +
+ " 10\n" +
+ " %s\n" +
+ "\n", port);
+ Files.writeString(tempFile.toPath(), string);
+ return tempFile.toPath();
+ }
+ catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public boolean isLatestDriverMinimumSupportedVersion()
+ {
+ return latestDriverMinimumSupportedVersion;
}
public void execute(String sql)
@@ -64,15 +109,26 @@ public void execute(String sql)
}
}
+ @Override
+ public String getDriverClassName()
+ {
+ if (latestDriverMinimumSupportedVersion) {
+ return CLICKHOUSE_LATEST_DRIVER_CLASS_NAME;
+ }
+
+ return CLICKHOUSE_DEPRECATED_DRIVER_CLASS_NAME;
+ }
+
+ @Override
public String getJdbcUrl()
{
- return format("jdbc:clickhouse://%s:%s/", dockerContainer.getContainerIpAddress(),
- dockerContainer.getMappedPort(HTTP_PORT));
+ return format("jdbc:clickhouse://%s:%s/%s", getContainerIpAddress(), getMappedPort(port),
+ CLICKHOUSE_DEPRECATED_DRIVER_CLASS_NAME.equals(getDriverClassName()) ? "?validateAfterInactivityMillis=100" : "");
}
@Override
public void close()
{
- dockerContainer.stop();
+ stop();
}
}
diff --git a/plugin/trino-clickhouse/src/test/resources/custom.xml b/plugin/trino-clickhouse/src/test/resources/custom.xml
index f4fa0e8b59d5..43ca98992beb 100644
--- a/plugin/trino-clickhouse/src/test/resources/custom.xml
+++ b/plugin/trino-clickhouse/src/test/resources/custom.xml
@@ -2,4 +2,5 @@
10
+ 9999