diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 49700cd17..66ecee8ff 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -26,7 +26,7 @@ on: driver: description: "Driver version" required: true - default: "0.3.1-SNAPSHOT" + default: "0.3.2-SNAPSHOT" options: description: "Benchmark options" required: true @@ -55,14 +55,14 @@ jobs: run: | find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ steps.version.outputs.value }}|g' \ -e 's|^\( \).*\(\)$|\1${{ steps.version.outputs.value }}\2|' \ - -e 's|${parent.groupId}|tech.clickhouse|g' -e 's|.*argLine.*timezone=.*||g' '{}' \; + -e 's|${parent.groupId}|com.clickhouse|g' '{}' \; find . -type f -name "log4j.*" -exec rm -fv '{}' \; - name: Build project run: | mvn --batch-mode --update-snapshots -q -DskipTests install cd clickhouse-benchmark mvn --batch-mode --update-snapshots install - java -DclickhouseVersion="21.3" -jar target/benchmarks.jar -rf text -p client=clickhouse-jdbc Basic + java -DclickhouseVersion="21.8" -jar target/benchmarks.jar -rf text -p client=clickhouse-jdbc Basic echo "BENCHMARK_REPORT<> $GITHUB_ENV cat jmh-result.text >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV @@ -99,7 +99,7 @@ jobs: run: | find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.version }}|g' \ -e 's|^\( \).*\(\)$|\1${{ github.event.inputs.driver }}\2|' \ - -e 's|${parent.groupId}|tech.clickhouse|g' -e 's|.*argLine.*timezone=.*||g' '{}' \; + -e 's|${parent.groupId}|com.clickhouse|g' '{}' \; find . -type f -name "log4j.*" -exec rm -fv '{}' \; continue-on-error: true - name: Install driver as needed @@ -110,7 +110,7 @@ jobs: run: | mvn --batch-mode --update-snapshots -DskipTests -pl clickhouse-benchmark -am package cd clickhouse-benchmark - java -DclickhouseVersion="21.3" -jar target/benchmarks.jar -rf json ${{ github.event.inputs.options }} > output.txt + java -DclickhouseVersion="21.8" -jar target/benchmarks.jar -rf json ${{ github.event.inputs.options }} > output.txt echo "BENCHMARK_REPORT<> $GITHUB_ENV tail -n +$(grep -n '^REMEMBER:' output.txt | tail -1 | awk -F: '{print $1+6}') output.txt | head -n -2 | grep -v ':·' >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 32ff1dc46..9ade21eba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,9 +32,9 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: [8, 11, 15] + java: [8, 11, 17] # most recent LTS releases as well as latest stable builds - clickhouse: ["20.8", "21.3", "latest"] + clickhouse: ["21.3", "21.8", "latest"] fail-fast: false name: Build using JDK ${{ matrix.java }} against ClickHouse ${{ matrix.clickhouse }} steps: @@ -59,4 +59,4 @@ jobs: ${{ runner.os }}-build- - name: Build with Maven run: | - mvn --batch-mode --update-snapshots -DclickhouseVersion=${{ matrix.clickhouse }} verify + mvn --batch-mode --update-snapshots -Drelease -DclickhouseVersion=${{ matrix.clickhouse }} verify diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f003027ef..cc7b0f51b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,7 @@ on: version: description: "Release version" required: true - default: "0.3.1-SNAPSHOT" + default: "0.3.2-SNAPSHOT" jobs: release: @@ -24,7 +24,7 @@ jobs: run: | find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.version }}|g' \ -e 's|^\( \).*\(\)$|\1${{ github.event.inputs.version }}\2|' \ - -e 's|${parent.groupId}|tech.clickhouse|g' -e 's|.*argLine.*timezone=.*||g' '{}' \; + -e 's|${parent.groupId}|com.clickhouse|g' '{}' \; find . -type f -name "log4j.*" -exec rm -fv '{}' \; - name: Release Maven package uses: samuelmeuli/action-maven-publish@v1 diff --git a/.github/workflows/third_party_libs.yml b/.github/workflows/third_party_libs.yml new file mode 100644 index 000000000..d33f50581 --- /dev/null +++ b/.github/workflows/third_party_libs.yml @@ -0,0 +1,38 @@ +name: ThirdPartyLibs + +on: + workflow_dispatch: + inputs: + version: + description: "Release version" + required: true + default: "1.0.0" + +jobs: + release: + name: "Build and Publish Repackaged 3rd Party Libraries" + runs-on: "ubuntu-latest" + + steps: + - name: Check out Git repository + uses: actions/checkout@v2 + - name: Install Java and Maven + uses: actions/setup-java@v1 + with: + java-version: 9 + - name: Update pom files and reduce logs + run: | + find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.version }}|g' \ + -e 's|^\( \).*\(\)$|\1${{ github.event.inputs.version }}\2|' \ + -e 's|${parent.groupId}|com.clickhouse|g' '{}' \; + find . -type f -name "log4j.*" -exec rm -fv '{}' \; + - name: Release Maven package + uses: samuelmeuli/action-maven-publish@v1 + with: + directory: third-party-libraries + maven_profiles: release + maven_args: -q --batch-mode + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }} + nexus_username: ${{ secrets.SONATYPE_USER }} + nexus_password: ${{ secrets.SONATYPE_PASSWD }} diff --git a/.github/workflows/timezone.yml b/.github/workflows/timezone.yml index 6134c5b15..3cc190dd6 100644 --- a/.github/workflows/timezone.yml +++ b/.github/workflows/timezone.yml @@ -56,8 +56,7 @@ jobs: ${{ runner.os }}-build- - name: Test using Maven run: | - find . -type f -name "pom.xml" -exec sed -i -e 's|.*argLine.*timezone=.*||g' '{}' \; mvn --batch-mode --update-snapshots \ -DclickhouseTimezone=${{ matrix.serverTz }} \ - -DclickhouseVersion=21.3 \ + -DclickhouseVersion=21.8 \ -Duser.timezone=${{ matrix.clientTz }} verify diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 86fef8a0d..91725e5fb 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -85,7 +85,6 @@ jobs: - name: Verify with Maven if: steps.check.outputs.triggered == 'true' run: | - find . -type f -name "pom.xml" -exec sed -i -e 's|.*argLine.*timezone=.*||g' '{}' \; mvn --batch-mode --update-snapshots \ -DclickhouseVersion=${{ fromJSON(steps.commented.outputs.result).clickhouse }} \ -DclickhouseTimezone=Asia/Chongqing \ @@ -129,7 +128,6 @@ jobs: continue-on-error: true - name: Verify with Maven run: | - find . -type f -name "pom.xml" -exec sed -i -e 's|.*argLine.*timezone=.*||g' '{}' \; mvn --batch-mode --update-snapshots \ -DclickhouseVersion=${{ github.event.inputs.clickhouse }} \ -DclickhouseTimezone=${{ github.event.inputs.chTz }} \ diff --git a/.gitignore b/.gitignore index 8e4ab4f9f..3109e7d25 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ -*.class +# MacOS +.DS_Store -# Package Files # +# Java Files +*.class *.jar *.war *.ear @@ -30,13 +32,19 @@ target/ # Generated files .flattened-pom.xml +dependency-reduced-pom.xml **/parser/*CharStream.java **/parser/ClickHouseSqlParser.java **/parser/ClickHouseSqlParserConstants.java **/parser/ClickHouseSqlParserTokenManager.java **/parser/Token*.java **/parser/ParseException.java +java.prof jmh-result.* +profile.html # Shell scripts *.sh + +# test configuration +**/test/resources/test.properties diff --git a/CHANGELOG b/CHANGELOG index e9c0034ed..5463a629f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +0.3.1-patch + * fix incorrect database used in DDL + * fix batch insert issue when no VALUES used in SQL statement + * fix issue of handling negative decimal128/256 values + 0.3.1 * BREAKING CHANGE - move query from url to request body * BREAKING CHANGE - always parse SQL(use extended API to skip that) @@ -10,6 +15,7 @@ * fix parser issue when DESC statement contains alias * support batch processing with arbitrary query - update and delete are not recommended so there'll be warnings * support multi-statement sql - session will be used automatically and only the last result will be returned + 0.3.0 * BREAKING CHANGE - dropped JDK 7 support * BREAKING CHANGE - removed Guava dependency(and so is UnsignedLong) @@ -23,6 +29,7 @@ * fix error when using ClickHouseCompression.none against 19.16 * fix NegativeArraySizeException when dealing with large array * fix datetime/date display issue caused by timezone differences(between client and column/server) + 0.2.6 * add new feature for sending compressed files/streams * introduce an experimental SQL parser to fix parsing related issues - set connection setting use_new_parser to false to disable @@ -33,6 +40,7 @@ * upgrade to lz4-java and improve performance of LZ4 stream * use HTTP Basic Auth for credentials instead of query parameters * use static version instead of property-based revision in pom.xml + 0.2.5 * bump dependencies and include lz4 in shaded jar * new API: ClickHouseRowBinaryStream.writeUInt64Array(UnsignedLong[]) @@ -42,20 +50,26 @@ * fix ResultSet.findColumn(String) issue * fix the issue of not being able to use NULL constant in PreparedStatement * fix toLowerCase issue for Turkish + 0.2.4 * fix FORMAT clause append for queries, ending with comment + 0.2.3 * added support for Decimals in RowBinary protocol + 0.2.2 * close certificate keystore * fix for Boolean data type + 0.2.1 * implement some ResultSet metadata methods * added support for "any_join_distinct_right_table_keys" setting * nested array support + 0.2 * new API for writing streams of data * deprecation of send* methods in ClickHouseStatement interface + 0.1.55 NOTE: behavior for byte[] parameters changed. See https://github.com/yandex/clickhouse-jdbc/pull/352 * support for sending stream of CSV data diff --git a/README.md b/README.md index b615e9ed8..8bc5d345b 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,13 @@ ClickHouse JDBC driver This is a basic and restricted implementation of jdbc driver for ClickHouse. It has support of a minimal subset of features to be usable. + ### Usage ```xml ru.yandex.clickhouse clickhouse-jdbc - 0.3.1 + 0.3.2 ``` @@ -46,6 +47,7 @@ try (ClickHouseConnection conn = dataSource.getConnection(); Additionally, if you have a few instances, you can use `BalancedClickhouseDataSource`. + ### Extended API In order to provide non-JDBC complaint data manipulation functionality, proprietary API exists. Entry point for API is `ClickHouseStatement#write()` method. @@ -61,6 +63,7 @@ sth .data(new File("/path/to/file.csv.gz"), ClickHouseFormat.CSV, ClickHouseCompression.gzip) // specify input .send(); ``` + #### Configurable send ```java import ru.yandex.clickhouse.ClickHouseStatement; @@ -73,6 +76,7 @@ sth .addDbParam(ClickHouseQueryParam.MAX_PARALLEL_REPLICAS, 2) .send(); ``` + #### Send data in binary formatted with custom user callback ```java import ru.yandex.clickhouse.ClickHouseStatement; @@ -88,6 +92,12 @@ sth.write().send("INSERT INTO test.writer", new ClickHouseStreamCallback() { }, ClickHouseFormat.RowBinary); // RowBinary or Native are supported ``` + + +### Supported Server Versions +All [active releases](../ClickHouse/pulls?q=is%3Aopen+is%3Apr+label%3Arelease) are supported. You can still use the driver for older versions like 18.14 or 19.16 but please keep in mind that they're no longer supported. + + ### Compiling with maven The driver is built with maven. `mvn package -DskipTests=true` @@ -96,5 +106,6 @@ To build a jar with dependencies use `mvn package assembly:single -DskipTests=true` + ### Build requirements -In order to build the jdbc client one need to have jdk 1.7 or higher. +In order to build the jdbc client one need to have jdk 1.8 or higher. diff --git a/clickhouse-benchmark/pom.xml b/clickhouse-benchmark/pom.xml index fc02afda9..09fd5be64 100644 --- a/clickhouse-benchmark/pom.xml +++ b/clickhouse-benchmark/pom.xml @@ -18,12 +18,12 @@ 1.4.4 - 2.7.2 - 8.0.23 - 2.5.4 - 1.15.2 + 2.7.3 + 8.0.25 + 2.5.6 + 42.2.22 UTF-8 - 1.27 + 1.32 benchmarks @@ -97,6 +97,17 @@ + + org.postgresql + postgresql + ${postgresql-driver.version} + + + * + * + + + org.testcontainers testcontainers diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java index 079001636..f6e6c118b 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java @@ -3,32 +3,27 @@ import java.sql.ResultSet; import java.sql.Statement; import java.util.Collections; -import java.util.Random; import org.openjdk.jmh.annotations.Benchmark; public class Basic extends JdbcBenchmark { @Benchmark - public int selectOneRandomNumber(ClientState state) throws Throwable { - final int num = new Random().nextInt(1000); + public int insertOneRandomNumber(ClientState state) throws Throwable { + final int num = state.getRandomSample(); - try (Statement stmt = executeQuery(state, "select ? as n", num)) { - ResultSet rs = stmt.getResultSet(); + return executeInsert(state, "insert into test_insert(i) values(?)", + Collections.enumeration(Collections.singletonList(new Object[] { num }))); + } - rs.next(); + @Benchmark + public int selectOneRandomNumber(ClientState state) throws Throwable { + final int num = state.getRandomSample(); - if (num != rs.getInt(1)) { + try (Statement stmt = executeQuery(state, "select ? as n", num); ResultSet rs = stmt.getResultSet();) { + if (!rs.next() || num != rs.getInt(1)) { throw new IllegalStateException(); } return num; } } - - @Benchmark - public int insertOneRandomNumber(ClientState state) throws Throwable { - final int num = new Random().nextInt(1000); - - return executeInsert(state, "insert into test_insert(i) values(?)", - Collections.enumeration(Collections.singletonList(new Object[] { num }))); - } } diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java index 13d787529..bf5d49f74 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java @@ -1,9 +1,12 @@ package tech.clickhouse.benchmark; import java.sql.Connection; +import java.sql.Driver; import java.sql.SQLException; import java.sql.Statement; import java.util.Properties; +import java.util.Random; + import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; @@ -13,27 +16,43 @@ @State(Scope.Thread) public class ClientState { - @Param(value = { "clickhouse4j", Constants.CLICKHOUSE_DRIVER, "clickhouse-native-jdbc-shaded", - "mariadb-java-client", "mysql-connector-java" }) + // sample size used in 10k query/insert + private final int SAMPLE_SIZE = Integer.parseInt(System.getProperty("sampleSize", "10000")); + // floating range(to reduce server-side cache hits) used in 10k query/insert + private final int FLOATING_RANGE = Integer.parseInt(System.getProperty("floatingRange", "100")); + + @Param(value = { "clickhouse4j", "clickhouse-jdbc", "clickhouse-native-jdbc-shaded", "mariadb-java-client", + "mysql-connector-java", "postgresql-jdbc" }) private String client; + @Param(value = { Constants.REUSE_CONNECTION, Constants.NEW_CONNECTION }) + private String connection; + @Param(value = { Constants.NORMAL_STATEMENT, Constants.PREPARED_STATEMENT }) private String statement; + private Driver driver; + private String url; private Connection conn; + private int randomSample; + private int randomNum; + @Setup(Level.Trial) public void doSetup(ServerState serverState) throws Exception { - JdbcDriver driver = JdbcDriver.from(client); + JdbcDriver jdbcDriver = JdbcDriver.from(client); try { - conn = ((java.sql.Driver) Class.forName(driver.getClassName()).getDeclaredConstructor().newInstance()) - .connect(String.format(driver.getUrlTemplate(), serverState.getHost(), - serverState.getPort(driver.getDefaultPort()), serverState.getDatabase(), - serverState.getUser(), serverState.getPassword()), new Properties()); + driver = (java.sql.Driver) Class.forName(jdbcDriver.getClassName()).getDeclaredConstructor().newInstance(); + url = String.format(jdbcDriver.getUrlTemplate(), serverState.getHost(), + serverState.getPort(jdbcDriver.getDefaultPort()), serverState.getDatabase(), serverState.getUser(), + serverState.getPassword()); + conn = driver.connect(url, new Properties()); try (Statement s = conn.createStatement()) { - s.execute("create table if not exists system.test_insert(i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory"); + s.execute("truncate table if exists system.test_insert"); + s.execute( + "create table if not exists system.test_insert(i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory"); } } catch (SQLException e) { e.printStackTrace(); @@ -43,17 +62,56 @@ public void doSetup(ServerState serverState) throws Exception { @TearDown(Level.Trial) public void doTearDown(ServerState serverState) throws SQLException { - try (Statement s = conn.createStatement()) { - s.execute("drop table if exists system.test_insert"); + if (conn != null) { + conn.close(); + } + } + + @Setup(Level.Iteration) + public void prepare() { + if (!Constants.REUSE_CONNECTION.equalsIgnoreCase(connection)) { + try { + conn = driver.connect(url, new Properties()); + } catch (SQLException e) { + throw new IllegalStateException("Failed to create new connection", e); + } } - conn.close(); + + Random random = new Random(); + randomSample = random.nextInt(SAMPLE_SIZE); + randomNum = random.nextInt(FLOATING_RANGE); + } + + @TearDown(Level.Iteration) + public void shutdown() { + if (!Constants.REUSE_CONNECTION.equalsIgnoreCase(connection)) { + try { + conn.close(); + } catch (SQLException e) { + throw new IllegalStateException("Failed to close connection", e); + } finally { + conn = null; + } + } + } + + public int getSampleSize() { + return SAMPLE_SIZE; + } + + public int getRandomSample() { + return randomSample; + } + + public int getRandomNumber() { + return randomNum; } - public Connection getConnection() { - return this.conn; + public Connection getConnection() throws SQLException { + return conn; } public boolean usePreparedStatement() { - return Constants.PREPARED_STATEMENT.equals(this.statement); + return Constants.PREPARED_STATEMENT.equalsIgnoreCase(this.statement); } } diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java index 27220a5ef..9b23e84b1 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java @@ -4,8 +4,6 @@ * Constant interface. */ public interface Constants { - public static final String CLICKHOUSE_DRIVER = "clickhouse-jdbc"; - public static final String DEFAULT_HOST = "127.0.0.1"; public static final String DEFAULT_DB = "system"; public static final String DEFAULT_USER = "default"; @@ -13,9 +11,13 @@ public interface Constants { public static final int GRPC_PORT = 9100; public static final int HTTP_PORT = 8123; - public static final int MYSQL_PORT = 3307; + public static final int MYSQL_PORT = 9004; public static final int NATIVE_PORT = 9000; + public static final int POSTGRESQL_PORT = 9005; public static final String NORMAL_STATEMENT = "normal"; public static final String PREPARED_STATEMENT = "prepared"; + + public static final String REUSE_CONNECTION = "reuse"; + public static final String NEW_CONNECTION = "new"; } diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java index b6b2cfe17..6501680d4 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java @@ -1,23 +1,14 @@ package tech.clickhouse.benchmark; import java.sql.Timestamp; -// import java.util.Collections; import java.util.Enumeration; -import java.util.Random; import org.openjdk.jmh.annotations.Benchmark; public class Insertion extends JdbcBenchmark { - // @Benchmark - // public int insertOneNumber(ClientState state) throws Throwable { - // return executeInsert(state, "insert into test_insert(i) values(?)", - // Collections.enumeration(Collections.singletonList(new Object[] { new - // Random().nextInt(1000) }))); - // } - @Benchmark public int insert10kUInt64Rows(ClientState state) throws Throwable { - final int rows = 10000; - final int num = new Random().nextInt(rows); + final int range = state.getRandomNumber(); + final int rows = state.getSampleSize() + range; return executeInsert(state, "insert into system.test_insert(i) values(?)", new Enumeration() { int counter = 0; @@ -29,15 +20,15 @@ public boolean hasMoreElements() { @Override public Object[] nextElement() { - return new Object[] { num + (counter++) }; + return new Object[] { range + (counter++) }; } }); } @Benchmark public int insert10kStringRows(ClientState state) throws Throwable { - final int rows = 10000; - final int num = new Random().nextInt(rows); + final int range = state.getRandomNumber(); + final int rows = state.getSampleSize() + range; return executeInsert(state, "insert into system.test_insert(s) values(?)", new Enumeration() { int counter = 0; @@ -49,15 +40,15 @@ public boolean hasMoreElements() { @Override public Object[] nextElement() { - return new Object[] { String.valueOf(num + (counter++)) }; + return new Object[] { String.valueOf(range + (counter++)) }; } }); } @Benchmark public int insert10kTimestampRows(ClientState state) throws Throwable { - final int rows = 10000; - final int num = new Random().nextInt(rows); + final int range = state.getRandomNumber(); + final int rows = state.getSampleSize() + range; return executeInsert(state, "insert into system.test_insert(t) values(?)", new Enumeration() { int counter = 0; @@ -69,7 +60,7 @@ public boolean hasMoreElements() { @Override public Object[] nextElement() { - return new Object[] { new Timestamp((long) num + (counter++)) }; + return new Object[] { new Timestamp((long) range + (counter++)) }; } }); } diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java index dac7ce228..39b1bc8f2 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java @@ -29,9 +29,9 @@ @OutputTimeUnit(TimeUnit.SECONDS) public abstract class JdbcBenchmark { // batch size for mutation - private final int batchSize = Integer.parseInt(System.getProperty("batchSize", "1000")); + private final int BATCH_SIZE = Integer.parseInt(System.getProperty("batchSize", "5000")); // fetch size for query - private final int fetchSize = Integer.parseInt(System.getProperty("fetchSize", "1000")); + private final int FETCH_SIZE = Integer.parseInt(System.getProperty("fetchSize", "1000")); protected PreparedStatement setParameters(PreparedStatement s, Object... values) throws SQLException { if (values != null && values.length > 0) { @@ -73,16 +73,30 @@ private int processBatch(Statement s, String sql, Enumeration generato while (generator.hasMoreElements()) { Object[] values = generator.nextElement(); if (ps != null) { - setParameters(ps, values).addBatch(); + setParameters(ps, values); + + if (BATCH_SIZE > 0) { + ps.addBatch(); + } else { + ps.execute(); + rows++; + } } else { - s.addBatch(replaceParameters(sql, values)); + sql = replaceParameters(sql, values); + if (BATCH_SIZE > 0) { + s.addBatch(sql); + } else { + s.execute(sql); + rows++; + } } - if (++counter % batchSize == 0) { + + if (BATCH_SIZE > 0 && ++counter % BATCH_SIZE == 0) { rows += s.executeBatch().length; } } - if (counter % batchSize != 0) { + if (BATCH_SIZE > 0 && counter % BATCH_SIZE != 0) { rows += s.executeBatch().length; } @@ -114,17 +128,14 @@ protected Statement executeQuery(ClientState state, String sql, Object... values final Connection conn = state.getConnection(); if (state.usePreparedStatement()) { - try (PreparedStatement s = conn.prepareStatement(sql)) { - stmt = s; - s.setFetchSize(fetchSize); - setParameters(s, values).executeQuery(); - } + PreparedStatement s = conn.prepareStatement(sql); + stmt = s; + s.setFetchSize(FETCH_SIZE); + setParameters(s, values).executeQuery(); } else { - try (Statement s = conn.createStatement()) { - stmt = s; - stmt.setFetchSize(fetchSize); - stmt.executeQuery(replaceParameters(sql, values)); - } + stmt = conn.createStatement(); + stmt.setFetchSize(FETCH_SIZE); + stmt.executeQuery(replaceParameters(sql, values)); } return stmt; diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java index 4e8111a4e..8656638d7 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java @@ -24,7 +24,12 @@ public enum JdbcDriver { MysqlConnectorJava("com.mysql.cj.jdbc.Driver", "jdbc:mysql://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false" + "&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC", - Constants.MYSQL_PORT); + Constants.MYSQL_PORT), + + // PostgreSQL JDBC Driver + PostgresqlJdbc("org.postgresql.Driver", + "jdbc:postgresql://%s:%s/%s?user=%s&password=%s&ssl=false&sslmode=disable&preferQueryMode=simple", + Constants.POSTGRESQL_PORT); private final String className; private final String urlTemplate; diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java index 020fa3087..1480a142b 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java @@ -2,22 +2,21 @@ import java.sql.ResultSet; import java.sql.Statement; -import java.sql.Timestamp; -import java.util.Random; import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.infra.Blackhole; public class Query extends JdbcBenchmark { @Benchmark - public int select10kUInt64Rows(ClientState state) throws Throwable { - int rows = 10000; - int num = new Random().nextInt(rows); - try (Statement stmt = executeQuery(state, "select * from system.numbers where number > ? limit " + rows, num)) { + public int select10kUInt64Rows(Blackhole blackhole, ClientState state) throws Throwable { + int num = state.getRandomNumber(); + int rows = state.getSampleSize() + num; + try (Statement stmt = executeQuery(state, "select * from system.numbers limit ?", rows)) { ResultSet rs = stmt.getResultSet(); - float avg = 0.0F; int count = 0; while (rs.next()) { - avg = (rs.getInt(1) + avg * count) / (++count); + blackhole.consume(rs.getInt(1)); + count++; } if (count != rows) { @@ -29,17 +28,15 @@ public int select10kUInt64Rows(ClientState state) throws Throwable { } @Benchmark - public int select10kStringRows(ClientState state) throws Throwable { - int rows = 10000; - int num = new Random().nextInt(rows); - try (Statement stmt = executeQuery(state, - "select toString(number) as s from system.numbers where number > ? limit " + rows, num)) { + public int select10kStringRows(Blackhole blackhole, ClientState state) throws Throwable { + int num = state.getRandomNumber(); + int rows = state.getSampleSize() + num; + try (Statement stmt = executeQuery(state, "select toString(number) as s from system.numbers limit ?", rows)) { ResultSet rs = stmt.getResultSet(); int count = 0; - String str = null; while (rs.next()) { - str = rs.getString(1); + blackhole.consume(rs.getString(1)); count++; } @@ -52,19 +49,16 @@ public int select10kStringRows(ClientState state) throws Throwable { } @Benchmark - public int select10kTimestampRows(ClientState state) throws Throwable { - int rows = 10000; - int num = new Random().nextInt(rows); + public int select10kTimestampRows(Blackhole blackhole, ClientState state) throws Throwable { + int num = state.getRandomNumber(); + int rows = state.getSampleSize() + num; try (Statement stmt = executeQuery(state, - "select toDateTime('2021-02-20 13:15:20') + number as d from system.numbers where number > ? limit " - + rows, - num)) { + "select toDateTime('2021-02-20 13:15:20') + number as d from system.numbers limit ?", rows)) { ResultSet rs = stmt.getResultSet(); int count = 0; - Timestamp ts = null; while (rs.next()) { - ts = rs.getTimestamp(1); + blackhole.consume(rs.getTimestamp(1)); count++; } diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/BalancedClickhouseDataSource.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/BalancedClickhouseDataSource.java index 8b1c00c42..3bf9a4b36 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/BalancedClickhouseDataSource.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/BalancedClickhouseDataSource.java @@ -27,7 +27,7 @@ public class BalancedClickhouseDataSource implements DataSource { private static final org.slf4j.Logger log = LoggerFactory.getLogger(BalancedClickhouseDataSource.class); private static final Pattern URL_TEMPLATE = Pattern.compile(JDBC_CLICKHOUSE_PREFIX + "" + - "//([a-zA-Z0-9_:,.-]+)" + + "//([a-zA-Z0-9_\\[\\]:,.-]+)" + "(/[a-zA-Z0-9_]+" + "([?][a-zA-Z0-9_]+[=][a-zA-Z0-9_]+([&][a-zA-Z0-9_]+[=][a-zA-Z0-9_]+)*)?" + ")?"); diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java index 1ba8d191e..70985779b 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java @@ -88,6 +88,9 @@ private void initConnection(ClickHouseProperties properties) throws SQLException } serverTimeZone = TimeZone.getTimeZone("UTC"); // just for next query + timezone = serverTimeZone; + serverVersion = ""; + try (Statement s = createStatement(); ResultSet rs = s.executeQuery("select timezone(), version()")) { if (rs.next()) { serverTimeZone = TimeZone.getTimeZone(rs.getString(1)); diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDatabaseMetadata.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDatabaseMetadata.java index 37e0f7795..31209d105 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDatabaseMetadata.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDatabaseMetadata.java @@ -801,12 +801,12 @@ private static void buildAndCondition(StringBuilder dest, List condition @Override public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { StringBuilder query; - if (connection.getServerVersion().compareTo("1.1.54237") > 0) { + if (ClickHouseVersionNumberUtil.compare(connection.getServerVersion(), "18.16") >= 0) { query = new StringBuilder( "SELECT database, table, name, type, default_kind as default_type, default_expression, comment "); } else { query = new StringBuilder( - "SELECT database, table, name, type, default_type, default_expression, NULL AS comment "); + "SELECT database, table, name, type, default_kind as default_type, default_expression, NULL AS comment "); } query.append("FROM system.columns "); List predicates = new ArrayList<>(); diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDriver.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDriver.java index a7673cafe..ab14046e1 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDriver.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDriver.java @@ -46,7 +46,7 @@ public class ClickHouseDriver implements Driver { } catch (SQLException e) { throw new RuntimeException(e); } - logger.info("Driver registered"); + logger.debug("Driver registered"); } @Override diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java index 47d25c406..98fce4ccf 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java @@ -738,7 +738,9 @@ private InputStream getInputStream( requestEntity = new StringEntity(sql, StandardCharsets.UTF_8); } else { MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); - entityBuilder.addTextBody("query", sql); + + ContentType queryContentType = ContentType.create(ContentType.TEXT_PLAIN.getMimeType(), StandardCharsets.UTF_8); + entityBuilder.addTextBody("query", sql, queryContentType); try { for (ClickHouseExternalData externalDataItem : externalData) { diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java index 773914c05..bd5d081c1 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java @@ -8,44 +8,110 @@ * doubt, just omit any specific format and let the driver take care of it. *

* - * @see ClickHouse Reference Documentation + * @see ClickHouse + * Reference Documentation * * @author Dmitry Andreev */ public enum ClickHouseFormat { - CustomSeparated, - TabSeparated, - TabSeparatedRaw, - TabSeparatedWithNames, - TabSeparatedWithNamesAndTypes, - CSV, - CSVWithNames, - Values, - Vertical, - JSON, - JSONCompact, - JSONCompactString, - JSONEachRow, - JSONStringEachRow, - JSONCompactEachRow, - JSONCompactStringEachRow, - TSKV, - TSV, - Pretty, - PrettyCompact, - PrettyCompactMonoBlock, - PrettyNoEscapes, - PrettySpace, - Protobuf, - RowBinary, - RowBinaryWithNamesAndTypes, - Native, - Null, - XML, - CapnProto, - Parquet, - ORC - ; + Arrow(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#arrow + ArrowStream(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#arrowstream + Avro(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#avro + AvroConfluent(true, false, true, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#avroconfluent + CapnProto(true, false, true, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#capnproto + CSV(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#csv + CSVWithNames(true, true, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#csvwithnames + CustomSeparated(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#customseparated + JSON(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#json + JSONAsString(true, false, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#jsonasstring + JSONCompact(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoncompact + JSONCompactEachRow(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoncompacteachrow + JSONCompactEachRowWithNamesAndTypes(true, true, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoncompacteachrowwithnamesandtypes + JSONCompactString(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoncompactstring + JSONCompactStringEachRow(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoncompactstringeachrow + JSONCompactStringEachRowWithNamesAndTypes(true, true, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoncompactstringeachrowwithnamesandtypes + JSONEachRow(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoneachrow + JSONEachRowWithProgress(false, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoneachrowwithprogress + JSONString(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#jsonstring + JSONStringsEachRow(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsonstringseachrow + JSONStringsEachRowWithProgress(false, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsonstringseachrowwithprogress + LineAsString(true, false, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#lineasstring + Native(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#native + Null(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#null + ORC(true, false, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#orc + Parquet(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#parquet + Pretty(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#pretty + PrettyCompact(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#prettycompact + PrettyCompactMonoBlock(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#prettycompactmonoblock + PrettyNoEscapes(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#prettynoescapes + PrettySpace(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#prettyspace + Protobuf(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#protobuf + ProtobufSingle(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#protobufsingle + RawBLOB(true, true, true, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#rawblob + Regexp(true, false, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#regexp + RowBinary(true, true, true, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#rowbinary + RowBinaryWithNamesAndTypes(true, true, true, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#rowbinarywithnamesandtypes + TabSeparated(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#tabseparated + TabSeparatedRaw(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#tabseparatedraw + TabSeparatedWithNames(true, true, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#tabseparatedwithnames + TabSeparatedWithNamesAndTypes(true, true, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#tabseparatedwithnamesandtypes + TSV(true, true, false, false, true), // alias of TabSeparated + TSVRaw(true, true, false, false, true), // alias of TabSeparatedRaw + TSVWithNames(true, true, false, true, true), // alias of TabSeparatedWithNames + TSVWithNamesAndTypes(true, true, false, true, true), // alias of TabSeparatedWithNamesAndTypes + Template(true, true, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#template + TemplateIgnoreSpaces(true, false, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#templateignorespaces + TSKV(true, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#tskv + Values(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#values + Vertical(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#vertical + VerticalRaw(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#verticalraw + XML(false, true, false, false, false); // https://clickhouse.tech/docs/en/interfaces/formats/#xml + + private boolean input; + private boolean output; + private boolean binary; + private boolean header; + private boolean rowBased; + + ClickHouseFormat(boolean input, boolean output, boolean binary, boolean header, boolean rowBased) { + this.input = input; + this.output = output; + this.binary = binary; + this.header = output && header; + this.rowBased = rowBased; + } + + public boolean supportsInput() { + return input; + } + + public boolean supportsOutput() { + return output; + } + + public boolean isBinary() { + return binary; + } + + public boolean isText() { + return !binary; + } + + public boolean hasHeader() { + return header; + } + + /** + * Check whether the format is row based(e.g. read/write by row), which is a + * very useful hint on how to process the data. + * + * @return true if the format is row based; false otherwise(e.g. column, + * document, or structured-object etc.) + */ + public boolean isRowBased() { + return rowBased; + } public static boolean containsFormat(String statement) { if (statement == null || statement.isEmpty()) { @@ -60,5 +126,4 @@ public static boolean containsFormat(String statement) { } return false; } - } diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseConnectionSettings.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseConnectionSettings.java index 2f9b53568..1b59f6d66 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseConnectionSettings.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseConnectionSettings.java @@ -19,14 +19,6 @@ public enum ClickHouseConnectionSettings implements DriverPropertyCreator { CHECK_FOR_REDIRECTS("check_for_redirects", false, "whether we should check for 307 redirect using GET before sending POST to given URL"), MAX_REDIRECTS("max_redirects", 5, "number of redirect checks before using last URL"), - /* - * - * */ - DATA_TRANSFER_TIMEOUT( "dataTransferTimeout", 10000, "Timeout for data transfer. " - + " socketTimeout + dataTransferTimeout is sent to ClickHouse as max_execution_time. " - + " ClickHouse rejects request execution if its time exceeds max_execution_time"), - - /** * for ConnectionManager */ diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java index 10da23b81..9e8d831d7 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java @@ -16,7 +16,6 @@ public class ClickHouseProperties { private int apacheBufferSize; private int socketTimeout; private int connectionTimeout; - private int dataTransferTimeout; private int timeToLiveMillis; private int defaultMaxPerRoute; private int maxTotal; @@ -110,7 +109,6 @@ public ClickHouseProperties(Properties info) { this.apacheBufferSize = (Integer)getSetting(info, ClickHouseConnectionSettings.APACHE_BUFFER_SIZE); this.socketTimeout = (Integer)getSetting(info, ClickHouseConnectionSettings.SOCKET_TIMEOUT); this.connectionTimeout = (Integer)getSetting(info, ClickHouseConnectionSettings.CONNECTION_TIMEOUT); - this.dataTransferTimeout = (Integer)getSetting(info, ClickHouseConnectionSettings.DATA_TRANSFER_TIMEOUT); this.timeToLiveMillis = (Integer)getSetting(info, ClickHouseConnectionSettings.TIME_TO_LIVE_MILLIS); this.defaultMaxPerRoute = (Integer)getSetting(info, ClickHouseConnectionSettings.DEFAULT_MAX_PER_ROUTE); this.maxTotal = (Integer)getSetting(info, ClickHouseConnectionSettings.MAX_TOTAL); @@ -178,7 +176,6 @@ public Properties asProperties() { ret.put(ClickHouseConnectionSettings.APACHE_BUFFER_SIZE.getKey(), String.valueOf(apacheBufferSize)); ret.put(ClickHouseConnectionSettings.SOCKET_TIMEOUT.getKey(), String.valueOf(socketTimeout)); ret.put(ClickHouseConnectionSettings.CONNECTION_TIMEOUT.getKey(), String.valueOf(connectionTimeout)); - ret.put(ClickHouseConnectionSettings.DATA_TRANSFER_TIMEOUT.getKey(), String.valueOf(dataTransferTimeout)); ret.put(ClickHouseConnectionSettings.TIME_TO_LIVE_MILLIS.getKey(), String.valueOf(timeToLiveMillis)); ret.put(ClickHouseConnectionSettings.DEFAULT_MAX_PER_ROUTE.getKey(), String.valueOf(defaultMaxPerRoute)); ret.put(ClickHouseConnectionSettings.MAX_TOTAL.getKey(), String.valueOf(maxTotal)); @@ -249,7 +246,6 @@ public ClickHouseProperties(ClickHouseProperties properties) { setApacheBufferSize(properties.apacheBufferSize); setSocketTimeout(properties.socketTimeout); setConnectionTimeout(properties.connectionTimeout); - setDataTransferTimeout(properties.dataTransferTimeout); setTimeToLiveMillis(properties.timeToLiveMillis); setDefaultMaxPerRoute(properties.defaultMaxPerRoute); setMaxTotal(properties.maxTotal); @@ -554,14 +550,6 @@ public void setConnectionTimeout(int connectionTimeout) { this.connectionTimeout = connectionTimeout; } - public int getDataTransferTimeout() { - return dataTransferTimeout; - } - - public void setDataTransferTimeout(int dataTransferTimeout) { - this.dataTransferTimeout = dataTransferTimeout; - } - public String getUser() { return user; } diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java index f33564093..96cc71c40 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java @@ -53,6 +53,8 @@ public enum ClickHouseQueryParam implements DriverPropertyCreator { FORCE_INDEX_BY_DATE("force_index_by_date", false, Boolean.class, ""), + FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS("force_optimize_skip_unused_shards", 0, Integer.class, "Enables or disables query execution if optimize_skip_unused_shards is enabled and skipping of unused shards is not possible. If the skipping is not possible and the setting is enabled, an exception will be thrown."), + FORCE_PRIMARY_KEY("force_primary_key", false, Boolean.class, ""), GLOBAL_SUBQUERIES_METHOD("global_subqueries_method", null, String.class, ""), @@ -181,6 +183,8 @@ public enum ClickHouseQueryParam implements DriverPropertyCreator { OPTIMIZE_MOVE_TO_PREWHERE("optimize_move_to_prewhere", true, Boolean.class, ""), + OPTIMIZE_SKIP_UNUSED_SHARDS("optimize_skip_unused_shards", 0, Integer.class, "Enables or disables skipping of unused shards for SELECT queries that have sharding key condition in WHERE/PREWHERE (assuming that the data is distributed by sharding key, otherwise does nothing)."), + OUTPUT_FORMAT_JSON_QUOTE_64BIT_INTEGERS("output_format_json_quote_64bit_integers", true, Boolean.class, "Controls quoting of 64-bit integers in JSON output format."), OUTPUT_FORMAT_PRETTY_MAX_ROWS("output_format_pretty_max_rows", null, Long.class, "Rows limit for Pretty formats."), diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseFormat.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseFormat.java deleted file mode 100644 index ac23cfd84..000000000 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseFormat.java +++ /dev/null @@ -1,13 +0,0 @@ -package ru.yandex.clickhouse.util; - -/** - * @author Dmitry Andreev - */ -public enum ClickHouseFormat { - TabSeparated, - TabSeparatedWithNamesAndTypes, - JSONCompact, - RowBinary, - Native, - CSV -} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStream.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStream.java index 626d84900..f5afcc0a0 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStream.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStream.java @@ -93,6 +93,33 @@ public void writeString(String string) throws IOException { out.write(bytes); } + /** + * Write string with predefined proper length. + * + * @param string Input string + * @throws IOException in case if an I/O error occurs + */ + public void writeFixedString(String string) throws IOException { + byte[] bytes = Objects.requireNonNull(string).getBytes(StandardCharsets.UTF_8); + out.write(bytes); + } + + /** + * Write string with any length, but it will be corrected, cut or extended to len + * + * @param string Input string + * @param len Length of FixedString + * @throws IOException in case if an I/O error occurs + */ + public void writeFixedString(String string, Integer len) throws IOException { + byte[] bytes = Objects.requireNonNull(string).getBytes(StandardCharsets.UTF_8); + Integer bl = bytes.length; + out.write(bytes, 0, Math.min(len, bl)); + for (int i = 0; i < len - bl; i++) { + out.write(0); + } + } + public void writeUInt8(boolean value) throws IOException { out.writeByte(value ? 1 : 0); } diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseVersionNumberUtil.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseVersionNumberUtil.java index eb6b20e9e..e3b0211c4 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseVersionNumberUtil.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseVersionNumberUtil.java @@ -9,8 +9,8 @@ */ public final class ClickHouseVersionNumberUtil { - private static final Pattern VERSION_NUMBER_PATTERN = - Pattern.compile("^\\s*(\\d+)\\.(\\d+).*"); + private static final Pattern VERSION_NUMBER_PATTERN = Pattern.compile("^\\s*(\\d+)\\.(\\d+).*"); + private static final Pattern NON_NUMBERIC_PATTERN = Pattern.compile("[^0-9.]"); public static int getMajorVersion(String versionNumber) { Matcher m = VERSION_NUMBER_PATTERN.matcher(versionNumber); @@ -22,5 +22,36 @@ public static int getMinorVersion(String versionNumber) { return m.matches() ? Integer.parseInt(m.group(2)) : 0; } - private ClickHouseVersionNumberUtil() { /* do not instantiate util */ } + public static int compare(String currentVersion, String targetVersion) { + if (currentVersion == null || targetVersion == null || currentVersion.isEmpty() || targetVersion.isEmpty()) { + throw new IllegalArgumentException("Both version cannot be null or empty"); + } + + currentVersion = NON_NUMBERIC_PATTERN.matcher(currentVersion).replaceAll(""); + targetVersion = NON_NUMBERIC_PATTERN.matcher(targetVersion).replaceAll(""); + if (currentVersion.equals(targetVersion)) { + return 0; + } + + String[] v1 = currentVersion.split("\\."); + String[] v2 = targetVersion.split("\\."); + + int result = 0; + for (int i = 0, len = Math.min(v1.length, v2.length); i < len; i++) { + int n1 = Integer.parseInt(v1[i]); + int n2 = Integer.parseInt(v2[i]); + + if (n1 == n2) { + continue; + } else { + result = n1 > n2 ? 1 : -1; + break; + } + } + + return result; + } + + private ClickHouseVersionNumberUtil() { + /* do not instantiate util */ } } diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/BalancedClickhouseDataSourceTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/BalancedClickhouseDataSourceTest.java index cbbab493a..d321f1bbd 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/BalancedClickhouseDataSourceTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/BalancedClickhouseDataSourceTest.java @@ -313,4 +313,18 @@ public void testConnectionWithAuth() throws SQLException { } } + @Test + public void testIPv6() throws Exception { + // dedup is not supported at all :< + assertEquals(Arrays.asList("jdbc:clickhouse://[::1]:12345", "jdbc:clickhouse://[0:0:0:0:0:0:0:1]:12345"), + BalancedClickhouseDataSource.splitUrl("jdbc:clickhouse://[::1]:12345,[0:0:0:0:0:0:0:1]:12345")); + assertEquals(Arrays.asList("jdbc:clickhouse://[192:168:0:0:0:0:0:1]:12345", "jdbc:clickhouse://[192:168:0:0:0:0:0:2]:12345"), + BalancedClickhouseDataSource.splitUrl("jdbc:clickhouse://[192:168:0:0:0:0:0:1]:12345,[192:168:0:0:0:0:0:2]:12345")); + + ClickHouseProperties properties = new ClickHouseProperties(); + String hostAddr = ClickHouseContainerForTest.getClickHouseHttpAddress(); + String ipAddr = ClickHouseContainerForTest.getClickHouseHttpAddress("[::1]"); + assertEquals(ClickHouseContainerForTest.newBalancedDataSource(properties, ipAddr).getConnection().getServerVersion(), + ClickHouseContainerForTest.newBalancedDataSource(properties, hostAddr).getConnection().getServerVersion()); + } } diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/ClickHouseContainerForTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/ClickHouseContainerForTest.java index 70a696f63..9aae3ebcc 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/ClickHouseContainerForTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/ClickHouseContainerForTest.java @@ -75,6 +75,12 @@ public static String getClickHouseHttpAddress(boolean useIPaddress) { .append(':').append(clickhouseContainer.getMappedPort(HTTP_PORT)).toString(); } + public static String getClickHouseHttpAddress(String customHostOrIp) { + return new StringBuilder() + .append(customHostOrIp == null || customHostOrIp.isEmpty() ? clickhouseContainer.getContainerIpAddress() : customHostOrIp) + .append(':').append(clickhouseContainer.getMappedPort(HTTP_PORT)).toString(); + } + public static ClickHouseDataSource newDataSource() { return newDataSource(new ClickHouseProperties()); } diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseDatabaseMetadataTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseDatabaseMetadataTest.java index 9b005be9e..e4e593f63 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseDatabaseMetadataTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseDatabaseMetadataTest.java @@ -24,7 +24,7 @@ public class ClickHouseDatabaseMetadataTest { private ClickHouseDataSource dataSource; - private Connection connection; + private ClickHouseConnection connection; @BeforeTest public void setUp() throws Exception { @@ -64,11 +64,14 @@ public void testMetadata() throws Exception { @Test public void testMetadataColumns() throws Exception { + boolean supportComment = ClickHouseVersionNumberUtil.compare(connection.getServerVersion(), "18.16") >= 0; connection.createStatement().executeQuery( "DROP TABLE IF EXISTS test.testMetadata"); connection.createStatement().executeQuery( "CREATE TABLE test.testMetadata(" - + "foo Float32, bar UInt8 DEFAULT 42 COMMENT 'baz') ENGINE = TinyLog"); + + "foo Float32, bar UInt8" + + (supportComment ? " DEFAULT 42 COMMENT 'baz'" : "") + + ") ENGINE = TinyLog"); ResultSet columns = connection.getMetaData().getColumns( null, "test", "testMetadata", null); columns.next(); @@ -97,8 +100,10 @@ public void testMetadataColumns() throws Exception { Assert.assertEquals(columns.getString("IS_AUTOINCREMENT"), "NO"); Assert.assertEquals(columns.getString("IS_GENERATEDCOLUMN"), "NO"); columns.next(); - Assert.assertEquals(columns.getInt("COLUMN_DEF"), 42); - Assert.assertEquals(columns.getObject("REMARKS"), "baz"); + if (supportComment) { + Assert.assertEquals(columns.getInt("COLUMN_DEF"), 42); + Assert.assertEquals(columns.getObject("REMARKS"), "baz"); + } } @Test diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseLargeNumberTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseLargeNumberTest.java index e0eb722e3..e9a393fe8 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseLargeNumberTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseLargeNumberTest.java @@ -27,9 +27,10 @@ import ru.yandex.clickhouse.except.ClickHouseException; import ru.yandex.clickhouse.settings.ClickHouseProperties; import ru.yandex.clickhouse.settings.ClickHouseQueryParam; +import ru.yandex.clickhouse.util.ClickHouseVersionNumberUtil; public class ClickHouseLargeNumberTest { - private Connection conn; + private ClickHouseConnection conn; @BeforeTest public void setUp() throws Exception { @@ -57,10 +58,10 @@ public void tearDown() throws Exception { @Test public void testBigIntSupport() throws SQLException { - if (conn == null) { + if (conn == null || ClickHouseVersionNumberUtil.compare(conn.getServerVersion(), "21.7") >= 0) { return; } - + String testSql = "create table if not exists system.test_bigint_support(i Int256) engine=Memory;" + "drop table if exists system.test_bigint_support;"; try (Connection conn = ClickHouseContainerForTest.newDataSource().getConnection(); diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseMapTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseMapTest.java index c056710a4..fcbdaa60e 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseMapTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseMapTest.java @@ -76,12 +76,17 @@ public void testMapSupport() throws SQLException { return; } + String testSql = "create table if not exists system.test_map_support(m Map(UInt8, String)) engine=Memory;" + "drop table if exists system.test_map_support;"; - try (Connection conn = ClickHouseContainerForTest.newDataSource().getConnection(); + try (ClickHouseConnection conn = ClickHouseContainerForTest.newDataSource().getConnection(); Statement s = conn.createStatement()) { s.execute("set allow_experimental_map_type=0;" + testSql); - fail("Should fail without enabling map support"); + + String version = conn.getServerVersion(); + if (version.compareTo("21.8") < 0) { + fail("Should fail without enabling map support"); + } } catch (SQLException e) { assertEquals(e.getErrorCode(), 44); } @@ -99,7 +104,10 @@ public void testMapSupport() throws SQLException { params.put(ClickHouseQueryParam.ALLOW_EXPERIMENTAL_MAP_TYPE, "0"); s.executeQuery(testSql, params); - fail("Should fail without enabling map support"); + String version = conn.getServerVersion(); + if (version.compareTo("21.8") < 0) { + fail("Should fail without enabling map support"); + } } catch (SQLException e) { assertEquals(e.getErrorCode(), 44); } diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseStatementImplTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseStatementImplTest.java index c1235bff5..cb41694ee 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseStatementImplTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseStatementImplTest.java @@ -10,14 +10,18 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.math.BigInteger; +import java.nio.charset.Charset; import java.sql.Array; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; +import java.util.ArrayList; import java.util.Collections; import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.CountDownLatch; @@ -250,6 +254,64 @@ public void testExternalDataStream() throws SQLException, UnsupportedEncodingExc Assert.assertEquals(cnt, 99500); } + @Test + public void testQueryWithMultipleExternalTables() throws SQLException { + int tables = 30; + int rows = 10; + + String ddl = "drop table if exists test.test_ext_data_query;\n" + + "create table test.test_ext_data_query (\n" + + " Cb String,\n" + + " CREATETIME DateTime64(3),\n" + + " TIMESTAMP UInt64,\n" + + " Cc String,\n" + + " Ca1 UInt64,\n" + + " Ca2 UInt64,\n" + + " Ca3 UInt64\n" + + ") engine = MergeTree()\n" + + "PARTITION BY toYYYYMMDD(CREATETIME)\n" + + "ORDER BY (Cb, CREATETIME, Cc);"; + + String template = "avgIf(Ca1, Cb in L%1$d) as avgCa1%2$d, sumIf(Ca1, Cb in L%1$d) as sumCa1%2$d, minIf(Ca1, Cb in L%1$d) as minCa1%2$d, maxIf(Ca1, Cb in L%1$d) as maxCa1%2$d, anyIf(Ca1, Cb in L%1$d) as anyCa1%2$d, avgIf(Ca2, Cb in L%1$d) as avgCa2%2$d, sumIf(Ca2, Cb in L%1$d) as sumCa2%2$d, minIf(Ca2, Cb in L%1$d) as minCa2%2$d, maxIf(Ca2, Cb in L%1$d) as maxCa2%2$d, anyIf(Ca2, Cb in L%1$d) as anyCa2%2$d, avgIf(Ca3, Cb in L%1$d) as avgCa3%2$d, sumIf(Ca3, Cb in L%1$d) as sumCa3%2$d, minIf(Ca3, Cb in L%1$d) as minCa3%2$d, maxIf(Ca3, Cb in L%1$d) as maxCa3%2$d, anyIf(Ca3, Cb in L%1$d) as anyCa3%2$d"; + ClickHouseProperties properties = new ClickHouseProperties(); + properties.setDatabase("test"); + properties.setSocketTimeout(300000); + properties.setMaxAstElements(Long.MAX_VALUE); + properties.setMaxTotal(20); + properties.setMaxQuerySize(104857600L); + + Map paramMap = new HashMap<>(); + paramMap.put(ClickHouseQueryParam.PRIORITY,"2"); + + StringBuilder sql = new StringBuilder().append("select "); + List extDataList = new ArrayList<>(tables); + for (int i = 0; i < tables; i++) { + sql.append(String.format(template, i, i + 1)).append(','); + List valueList = new ArrayList<>(rows); + for (int j = i, size = i + rows; j < size; j++) { + valueList.add(String.valueOf(j)); + } + String dnExtString = String.join("\n", valueList); + InputStream inputStream = new ByteArrayInputStream(dnExtString.getBytes(Charset.forName("UTF-8"))); + ClickHouseExternalData extData = new ClickHouseExternalData("L" + i, inputStream); + extData.setStructure("Cb String"); + extDataList.add(extData); + } + + if (tables > 0) { + sql.deleteCharAt(sql.length() - 1); + } else { + sql.append('*'); + } + sql.append(" from test.test_ext_data_query where TIMESTAMP >= 1625796480 and TIMESTAMP < 1625796540 and Cc = 'eth0'"); + + try (ClickHouseConnection c = ClickHouseContainerForTest.newDataSource(properties).getConnection(); + ClickHouseStatement s = c.createStatement();) { + s.execute(ddl); + ResultSet rs = s.executeQuery(sql.toString(), paramMap, extDataList); + assertTrue(tables <= 0 || rs.next()); + } + } @Test public void testResultSetWithExtremes() throws SQLException { diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java index ee7d6a7e8..84dc4892a 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java @@ -33,7 +33,12 @@ public void testLowCardinality() throws Exception{ final ClickHouseStatement statement = connection.createStatement(); connection.createStatement().execute("DROP TABLE IF EXISTS test.low_cardinality"); connection.createStatement().execute( - "CREATE TABLE test.low_cardinality (date Date, lowCardinality LowCardinality(String), string String) ENGINE = MergeTree(date, (date), 8192)" + "CREATE TABLE test.low_cardinality (date Date, " + + "lowCardinality LowCardinality(String), " + + "string String," + + "fixedString FixedString(3)," + + "fixedStringLC LowCardinality(FixedString(6))" + + ") ENGINE = MergeTree partition by toYYYYMM(date) order by date" ); // Code: 368, e.displayText() = DB::Exception: Bad cast from type DB::ColumnString to DB::ColumnLowCardinality @@ -44,11 +49,11 @@ public void testLowCardinality() throws Exception{ final Date date1 = new Date(1497474018000L); statement.sendNativeStream( - "INSERT INTO test.low_cardinality (date, lowCardinality, string)", + "INSERT INTO test.low_cardinality (date, lowCardinality, string, fixedString, fixedStringLC)", new ClickHouseStreamCallback() { @Override public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { - stream.writeUnsignedLeb128(3); // Columns number + stream.writeUnsignedLeb128(5); // Columns number stream.writeUnsignedLeb128(1); // Rows number stream.writeString("date"); // Column name @@ -62,6 +67,14 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { stream.writeString("string"); // Column name stream.writeString("String"); // Column type stream.writeString("string"); // value + + stream.writeString("fixedString"); // Column name + stream.writeString("FixedString(3)"); // Column type + stream.writeFixedString("str"); // value + + stream.writeString("fixedStringLC"); // Column name + stream.writeString("FixedString(6)"); // Column type + stream.writeFixedString("str1", 6); // value } } ); @@ -71,5 +84,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { Assert.assertTrue(rs.next()); assertEquals(rs.getString("lowCardinality"), "string"); assertEquals(rs.getString("string"), "string"); + assertEquals(rs.getString("fixedString"), "str"); + assertEquals(rs.getString("fixedStringLC"), "str1\0\0"); } } diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java index b6a672f89..bc4d490c5 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java @@ -83,8 +83,9 @@ private void createTable(String table) throws SQLException { "float32Array Array(Float32), " + "float64Array Array(Float64), " + "uuid UUID," + - "lowCardinality LowCardinality(String)" + - ") ENGINE = MergeTree(date, (date), 8192)" + "lowCardinality LowCardinality(String)," + + "fixedString FixedString(15)" + + ") ENGINE = MergeTree partition by toYYYYMM(date) order by date" ); } @@ -340,7 +341,10 @@ private void testRowBinaryStream(boolean rowBinaryResult) throws Exception { statement.sendRowBinaryStream( "INSERT INTO test.raw_binary " + - "(date, dateTime, string, int8, uInt8, int16, uInt16, int32, uInt32, int64, uInt64, float32, float64, dateArray, dateTimeArray, stringArray, int8Array, uInt8Array, int16Array, uInt16Array, int32Array, uInt32Array, int64Array, uInt64Array, float32Array, float64Array, uuid, lowCardinality)", + "(date, dateTime, string, int8, uInt8, int16, uInt16, int32, uInt32, int64, uInt64, float32, " + + "float64, dateArray, dateTimeArray, stringArray, int8Array, uInt8Array, int16Array, uInt16Array, " + + "int32Array, uInt32Array, int64Array, uInt64Array, float32Array, float64Array, uuid, lowCardinality, " + + "fixedString)", new ClickHouseStreamCallback() { @Override public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { @@ -373,6 +377,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { stream.writeFloat64Array(float64s1); stream.writeUUID(uuid1); stream.writeString("lowCardinality\n1"); + stream.writeFixedString("fixedString1\0\0\0"); stream.writeDate(date2); stream.writeDateTime(date2); @@ -402,6 +407,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { stream.writeFloat64Array(new double[]{}); stream.writeUUID(uuid2); stream.writeString("lowCardinality\n2"); + stream.writeFixedString("fixedString2", 15); } } ); @@ -427,6 +433,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { assertEquals(rs.getDouble("float64"), 42.21); assertEquals(rs.getObject("uuid").toString(), "123e4567-e89b-12d3-a456-426655440000"); assertEquals(rs.getString("lowCardinality"), "lowCardinality\n1"); + assertEquals(rs.getString("fixedString"), "fixedString1\0\0\0"); Date[] expectedDates1 = new Date[dates1.length]; for (int i = 0; i < dates1.length; i++) { @@ -470,6 +477,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { assertEquals(rs.getDouble("float64"), 77.77); assertEquals(rs.getString("uuid"), "789e0123-e89b-12d3-a456-426655444444"); assertEquals(rs.getString("lowCardinality"), "lowCardinality\n2"); + assertEquals(rs.getString("fixedString"), "fixedString2\0\0\0"); Assert.assertFalse(rs.next()); } else { @@ -518,6 +526,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { assertEquals(is.readUUID(), uuid1); assertEquals(is.readString(), "lowCardinality\n1"); + assertEquals(is.readFixedString(15), "fixedString1\0\0\0"); assertEquals(is.readDate(), withTimeAtStartOfDay(date2)); assertEquals(is.readDateTime().getTime(), date2.getTime()); @@ -550,6 +559,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { assertEquals(is.readUUID(), uuid2); assertEquals(is.readString(), "lowCardinality\n2"); + assertEquals(is.readFixedString(15), "fixedString2\0\0\0"); // check EOF try { diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStreamTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStreamTest.java index b7469055d..fe0dd8b9f 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStreamTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStreamTest.java @@ -262,6 +262,80 @@ public void write(ClickHouseRowBinaryStream stream) throws Exception { ); } + @Test + public void testFixedString() throws Exception { + check( + new StreamWriter() { + @Override + public void write(ClickHouseRowBinaryStream stream) throws Exception { + stream.writeFixedString( + "aaaa~����%20�&zzzzz" + ); + } + }, + new byte[]{ + 97, 97, 97, 97, 126, -17, -65, -67, -17, -65, -67, -17, -65, -67, -17, + -65, -67, 37, 50, 48, -17, -65, -67, 38, 122, 122, 122, 122, 122 + } + //clickhouse-client -q "select toFixedString('aaaa~����%20�&zzzzz', 29) format RowBinary"| od -vAn -td1 + ); + } + + @Test + public void testFixedStringLen() throws Exception { + check( + new StreamWriter() { + @Override + public void write(ClickHouseRowBinaryStream stream) throws Exception { + stream.writeFixedString( + "aaaa~����%20�&zzzzz", 32 + ); + } + }, + new byte[]{ + 97, 97, 97, 97, 126, -17, -65, -67, -17, -65, -67, -17, -65, -67, -17, + -65, -67, 37, 50, 48, -17, -65, -67, 38, 122, 122, 122, 122, 122, 0, 0, 0 + } + //clickhouse-client -q "select toFixedString('aaaa~����%20�&zzzzz', 32) format RowBinary"| od -vAn -td1 + ); + } + + @Test + public void testFixedStringLen1() throws Exception { + check( + new StreamWriter() { + @Override + public void write(ClickHouseRowBinaryStream stream) throws Exception { + stream.writeFixedString( + "", 5 + ); + } + }, + new byte[]{ + 0, 0, 0, 0, 0 + } + //clickhouse-client -q "select toFixedString('', 5) format RowBinary"| od -vAn -td1 + ); + } + + @Test + public void testFixedStringLen2() throws Exception { + check( + new StreamWriter() { + @Override + public void write(ClickHouseRowBinaryStream stream) throws Exception { + stream.writeFixedString( + "1234567890", 5 + ); + } + }, + new byte[]{ + 49, 50, 51, 52, 53 + } + //clickhouse-client -q "select toFixedString('1234567890', 5) format RowBinary"| od -vAn -td1 + ); + } + @Test public void testUUID() throws Exception { check( diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseVersionNumberUtilTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseVersionNumberUtilTest.java index 8da439ca2..50d152ec3 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseVersionNumberUtilTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseVersionNumberUtilTest.java @@ -10,7 +10,8 @@ public void testMajorNull() { try { ClickHouseVersionNumberUtil.getMajorVersion(null); Assert.fail(); - } catch (NullPointerException npe) { /* expected */ } + } catch (NullPointerException npe) { + /* expected */ } } @Test @@ -18,7 +19,8 @@ public void testMinorNull() { try { ClickHouseVersionNumberUtil.getMinorVersion(null); Assert.fail(); - } catch (NullPointerException npe) { /* expected */ } + } catch (NullPointerException npe) { + /* expected */ } } @Test @@ -62,4 +64,15 @@ public void testMinorSimple() { Assert.assertEquals(ClickHouseVersionNumberUtil.getMinorVersion("1.1-SNAPSHOT"), 1); } + @Test + public void testCompare() { + Assert.assertEquals(ClickHouseVersionNumberUtil.compare("1", "1"), 0); + Assert.assertEquals(ClickHouseVersionNumberUtil.compare("21.3", "21.12"), -1); + Assert.assertEquals(ClickHouseVersionNumberUtil.compare("21 .3", "21. 12"), -1); + Assert.assertEquals(ClickHouseVersionNumberUtil.compare("21 .3", "21. 12-test"), -1); + Assert.assertEquals(ClickHouseVersionNumberUtil.compare("21.3", "21.1"), 1); + Assert.assertEquals(ClickHouseVersionNumberUtil.compare("21. 3", "21 .1"), 1); + Assert.assertEquals(ClickHouseVersionNumberUtil.compare("21. 3-test", "21 .1"), 1); + Assert.assertEquals(ClickHouseVersionNumberUtil.compare("18.16", "18.16.0"), 0); + } } diff --git a/pom.xml b/pom.xml index 194135348..6a2728a88 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,5 @@ - + 4.0.0 tech.clickhouse @@ -74,7 +72,7 @@ - 0.3.1-SNAPSHOT + 0.3.2-SNAPSHOT 2021 UTF-8 UTF-8 @@ -82,14 +80,13 @@ 3.0.0 3.4.2 3.3.1 - 1.7.1 - 2.9.10 - 2.9.10.8 - 0.9.10 - 1.7.30 + 1.8.0 + 2.12.3 + 0.9.15 + 1.7.31 1.10.19 2.27.2 - 1.15.2 + 1.15.3 6.14.3 3.3.0 3.0.0-M1 @@ -113,7 +110,7 @@ ${project.artifactId} clickhouse https://sonarcloud.io - 0.3.1 + 0.3.2 @@ -121,12 +118,12 @@ com.fasterxml.jackson.core jackson-core - ${jackson-core.version} + ${jackson.version} com.fasterxml.jackson.core jackson-databind - ${jackson-databind.version} + ${jackson.version} com.github.ben-manes.caffeine @@ -251,6 +248,7 @@ ${failsafe-plugin.version} **/*.java + ${skipTests} ${skipITs} false @@ -271,6 +269,8 @@ maven-surefire-plugin ${surefire-plugin.version} + + ${skipUTs} false @@ -307,6 +307,10 @@ org.codehaus.mojo flatten-maven-plugin + + org.apache.maven.plugins + maven-failsafe-plugin + diff --git a/third-party-libraries/io.grpc/pom.xml b/third-party-libraries/io.grpc/pom.xml new file mode 100644 index 000000000..344831376 --- /dev/null +++ b/third-party-libraries/io.grpc/pom.xml @@ -0,0 +1,140 @@ + + + 4.0.0 + + + com.clickhouse + third-party-libraries + ${revision} + + + io.grpc + ${revision} + jar + + ${project.artifactId} + Repackaged grpc-java for JPMS support + https://github.com/ClickHouse/clickhouse-jdbc/tree/master/third-party-libraries/io.grpc + + + + io.grpc + grpc-api + + + io.grpc + grpc-core + + + com.google.code.gson + gson + + + + + io.grpc + grpc-context + + + io.grpc + grpc-netty-shaded + provided + + + io.grpc + grpc-okhttp + provided + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + io.opencensus + opencensus-api + + + io.opencensus + opencensus-impl + + + io.opencensus + opencensus-contrib-grpc-metrics + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + io/grpc/**/*.class + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade + package + + shade + + + false + true + false + true + + + *:* + + module-info.class + META-INF/MANIFEST.MF + android/** + com/lmax/** + google/** + io/perfmark/** + org/** + javax/** + + + + + + + + + + + + + org.moditect + moditect-maven-plugin + + + add-module-infos + package + + add-module-info + + + true + + src/main/java/module-info.java + + + + + + + + \ No newline at end of file diff --git a/third-party-libraries/io.grpc/src/main/java/io/grpc/LoadBalancerProvider.java b/third-party-libraries/io.grpc/src/main/java/io/grpc/LoadBalancerProvider.java new file mode 100644 index 000000000..b2ad3e33b --- /dev/null +++ b/third-party-libraries/io.grpc/src/main/java/io/grpc/LoadBalancerProvider.java @@ -0,0 +1,13 @@ +package io.grpc; + +/** + * Dummy class as placeholder. It will be removed during package phase. + */ +public class LoadBalancerProvider { + /** + * Default constructor. + */ + public LoadBalancerProvider() { + // dummy constructor + } +} diff --git a/third-party-libraries/io.grpc/src/main/java/io/grpc/ManagedChannelProvider.java b/third-party-libraries/io.grpc/src/main/java/io/grpc/ManagedChannelProvider.java new file mode 100644 index 000000000..ab29593c8 --- /dev/null +++ b/third-party-libraries/io.grpc/src/main/java/io/grpc/ManagedChannelProvider.java @@ -0,0 +1,13 @@ +package io.grpc; + +/** + * Dummy class as placeholder. It will be removed during package phase. + */ +public class ManagedChannelProvider { + /** + * Default constructor. + */ + public ManagedChannelProvider() { + // dummy constructor + } +} diff --git a/third-party-libraries/io.grpc/src/main/java/io/grpc/NameResolverProvider.java b/third-party-libraries/io.grpc/src/main/java/io/grpc/NameResolverProvider.java new file mode 100644 index 000000000..cd4e3d489 --- /dev/null +++ b/third-party-libraries/io.grpc/src/main/java/io/grpc/NameResolverProvider.java @@ -0,0 +1,13 @@ +package io.grpc; + +/** + * Dummy class as placeholder. It will be removed during package phase. + */ +public class NameResolverProvider { + /** + * Default constructor. + */ + public NameResolverProvider() { + // dummy constructor + } +} diff --git a/third-party-libraries/io.grpc/src/main/java/io/grpc/ServerProvider.java b/third-party-libraries/io.grpc/src/main/java/io/grpc/ServerProvider.java new file mode 100644 index 000000000..24a8e249a --- /dev/null +++ b/third-party-libraries/io.grpc/src/main/java/io/grpc/ServerProvider.java @@ -0,0 +1,13 @@ +package io.grpc; + +/** + * Dummy class as placeholder. It will be removed during package phase. + */ +public class ServerProvider { + /** + * Default constructor. + */ + public ServerProvider() { + // dummy constructor + } +} diff --git a/third-party-libraries/io.grpc/src/main/java/module-info.java b/third-party-libraries/io.grpc/src/main/java/module-info.java new file mode 100644 index 000000000..e699c8985 --- /dev/null +++ b/third-party-libraries/io.grpc/src/main/java/module-info.java @@ -0,0 +1,18 @@ +module io.grpc { + exports io.grpc; + // exports io.grpc.inprocess; + // exports io.grpc.internal; + // exports io.grpc.util; + + requires java.logging; + requires java.naming; + // requires com.google.errorprone.annotations; + // requires com.lmax.disruptor; + // requires io.perfmark; + + uses io.grpc.ManagedChannelProvider; + uses io.grpc.NameResolverProvider; + uses io.grpc.ServerProvider; + // uses io.grpc.internal.BinaryLogProvider; + uses io.grpc.LoadBalancerProvider; +} diff --git a/third-party-libraries/org.roaringbitmap/pom.xml b/third-party-libraries/org.roaringbitmap/pom.xml new file mode 100644 index 000000000..f85521e67 --- /dev/null +++ b/third-party-libraries/org.roaringbitmap/pom.xml @@ -0,0 +1,88 @@ + + 4.0.0 + + + com.clickhouse + third-party-libraries + ${revision} + + + org.roaringbitmap + ${revision} + jar + + ${project.artifactId} + Repackaged RoaringBitmap for JPMS support + https://github.com/ClickHouse/clickhouse-jdbc/tree/master/third-party-libraries/org.roaringbitmap + + + + org.roaringbitmap + RoaringBitmap + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + org/roaringbitmap/**/*.class + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade + package + + shade + + + false + true + false + true + + + *:* + + module-info.class + META-INF/MANIFEST.MF + + + + + + + + + + + + org.moditect + moditect-maven-plugin + + + add-module-infos + package + + add-module-info + + + true + + src/main/java/module-info.java + + + + + + + + \ No newline at end of file diff --git a/third-party-libraries/org.roaringbitmap/src/main/java/module-info.java b/third-party-libraries/org.roaringbitmap/src/main/java/module-info.java new file mode 100644 index 000000000..16a78c24f --- /dev/null +++ b/third-party-libraries/org.roaringbitmap/src/main/java/module-info.java @@ -0,0 +1,5 @@ +module org.roaringbitmap { + exports org.roaringbitmap; + exports org.roaringbitmap.buffer; + exports org.roaringbitmap.longlong; +} diff --git a/third-party-libraries/org.roaringbitmap/src/main/java/org/roaringbitmap/DummyClass.java b/third-party-libraries/org.roaringbitmap/src/main/java/org/roaringbitmap/DummyClass.java new file mode 100644 index 000000000..292d72ab1 --- /dev/null +++ b/third-party-libraries/org.roaringbitmap/src/main/java/org/roaringbitmap/DummyClass.java @@ -0,0 +1,12 @@ +package org.roaringbitmap; + +/** + * Dummy class as placeholder. It will be removed during package phase. + */ +public class DummyClass { + /** + * Default constructor. + */ + public DummyClass() { + } +} diff --git a/third-party-libraries/org.roaringbitmap/src/main/java/org/roaringbitmap/buffer/DummyClass.java b/third-party-libraries/org.roaringbitmap/src/main/java/org/roaringbitmap/buffer/DummyClass.java new file mode 100644 index 000000000..c47b4a285 --- /dev/null +++ b/third-party-libraries/org.roaringbitmap/src/main/java/org/roaringbitmap/buffer/DummyClass.java @@ -0,0 +1,12 @@ +package org.roaringbitmap.buffer; + +/** + * Dummy class as placeholder. It will be removed during package phase. + */ +public class DummyClass { + /** + * Default constructor. + */ + public DummyClass() { + } +} diff --git a/third-party-libraries/org.roaringbitmap/src/main/java/org/roaringbitmap/longlong/DummyClass.java b/third-party-libraries/org.roaringbitmap/src/main/java/org/roaringbitmap/longlong/DummyClass.java new file mode 100644 index 000000000..95b957cc7 --- /dev/null +++ b/third-party-libraries/org.roaringbitmap/src/main/java/org/roaringbitmap/longlong/DummyClass.java @@ -0,0 +1,12 @@ +package org.roaringbitmap.longlong; + +/** + * Dummy class as placeholder. It will be removed during package phase. + */ +public class DummyClass { + /** + * Default constructor. + */ + public DummyClass() { + } +} diff --git a/third-party-libraries/pom.xml b/third-party-libraries/pom.xml new file mode 100644 index 000000000..b076f683c --- /dev/null +++ b/third-party-libraries/pom.xml @@ -0,0 +1,329 @@ + + + 4.0.0 + + com.clickhouse + third-party-libraries + ${revision} + pom + + ${project.artifactId} + Repackaged third party libraries for JPMS compatibility + https://github.com/ClickHouse/clickhouse-jdbc/tree/master/third-party-libraries + 2021 + + + ClickHouse, Inc. + https://clickhouse.com/ + + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0 + repo + + + + + + zhicwu + Zhichun Wu + zhicwu@gmail.com + +8 + + + + + io.grpc + org.roaringbitmap + + + + https://github.com/ClickHouse/clickhouse-jdbc + scm:git@github.com:ClickHouse/clickhouse-jdbc.git + scm:git@github.com:ClickHouse/clickhouse-jdbc.git + HEAD + + + + Github + https://github.com/ClickHouse/clickhouse-jdbc/issues + + + + Github + https://github.com/ClickHouse/clickhouse-jdbc/actions + + + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + + 1.0.0 + 2021 + UTF-8 + UTF-8 + 1.40.1 + 2.8.8 + 0.28.3 + 0.9.21 + 3.3.0 + 3.0.0-M1 + 1.2.7 + 1.6.8 + 1.6 + 3.8.1 + 1.0.0.RC1 + 3.2.4 + 3.2.1 + 3.2.0 + 3.2.0 + + + + + + com.google.code.gson + gson + ${gson.version} + + + org.roaringbitmap + RoaringBitmap + ${roaring-bitmap.version} + + + io.grpc + grpc-api + ${grpc.version} + + + io.grpc + grpc-core + ${grpc.version} + + + io.grpc + grpc-context + ${grpc.version} + + + io.grpc + grpc-netty-shaded + ${grpc.version} + + + io.grpc + grpc-okhttp + ${grpc.version} + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + io.opencensus + opencensus-api + ${opencensus.version} + + + io.opencensus + opencensus-impl + ${opencensus.version} + + + io.opencensus + opencensus-contrib-grpc-metrics + ${opencensus.version} + + + org.apache.tomcat + annotations-api + ${annotations-api.version} + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${compiler-plugin.version} + + 9 + true + + -Xlint:all + -Werror + + + + + org.apache.maven.plugins + maven-deploy-plugin + ${deploy-plugin.version} + + + org.apache.maven.plugins + maven-gpg-plugin + ${gpg-plugin.version} + + + org.apache.maven.plugins + maven-jar-plugin + ${jar-plugin.version} + + + org.apache.maven.plugins + maven-javadoc-plugin + ${javadoc-plugin.version} + + + org.apache.maven.plugins + maven-shade-plugin + ${shade-plugin.version} + + + org.apache.maven.plugins + maven-source-plugin + ${source-plugin.version} + + + org.sonatype.plugins + nexus-staging-maven-plugin + ${staging-plugin.version} + + + org.codehaus.mojo + flatten-maven-plugin + ${flatten-plugin.version} + + true + ossrh + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + org.moditect + moditect-maven-plugin + ${moditect-plugin.version} + + + + + + org.codehaus.mojo + flatten-maven-plugin + + + + + + + release + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + + + --pinentry-mode + loopback + + + + + sign-artifacts + verify + + sign + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + org.sonatype.plugins + nexus-staging-maven-plugin + + + default-deploy + deploy + + deploy + + + + + ossrh + https://s01.oss.sonatype.org/ + + + + + + + \ No newline at end of file