diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml deleted file mode 100644 index 91725e5fb..000000000 --- a/.github/workflows/verify.yml +++ /dev/null @@ -1,158 +0,0 @@ -name: Verify - -on: - workflow_dispatch: - inputs: - clickhouse: - description: "ClickHouse version" - required: true - default: "latest" - java: - description: "Java version" - required: true - default: "8" - chTz: - description: "ClickHouse timezone" - required: true - default: "Asia/Chongqing" - javaTz: - description: "Java timezone" - required: true - default: "America/Los_Angeles" - pr: - description: "Pull request#" - required: false - pull_request: - types: - - opened - - synchronize - - reopened - paths-ignore: - - "**.md" - - "docs/**" - - "**/CHANGELOG" - issue_comment: - types: - - created - - edited - -jobs: - verify-commented-pr: - runs-on: ubuntu-latest - name: Verify commented PR - if: github.event_name == 'issue_comment' && github.event.issue.pull_request - steps: - - uses: zhicwu/pull-request-comment-trigger@master - id: check - with: - trigger: '@verify' - reaction: rocket - env: - GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' - - name: Extra parameters from PR comment - if: steps.check.outputs.triggered == 'true' - uses: actions/github-script@v3 - id: commented - env: - COMMENT_BODY: '${{ github.event.comment.body }}' - with: - script: | - const keyword = '@verify'; - let buildArgs = process.env.COMMENT_BODY; - core.info(`Got commented body: ${buildArgs}`); - buildArgs = buildArgs.substring(buildArgs.lastIndexOf(keyword) + keyword.length); - const args = buildArgs.match(/[^\s]+/g); - core.info(`Got commented arguments: ${args}`); - - return { - pr: context.issue.number, - clickhouse: args && args.length > 0 ? args[0] : "latest", - java: args && args.length > 1 ? args[1] : "8" - }; - - name: Check out repository - uses: actions/checkout@v2 - - name: Check out commented PR - if: steps.check.outputs.triggered == 'true' - run: | - git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \ - origin pull/${{ fromJSON(steps.commented.outputs.result).pr }}/merge:merged-pr && git checkout merged-pr - - name: Set up JDK - if: steps.check.outputs.triggered == 'true' - uses: actions/setup-java@v1 - with: - java-version: ${{ fromJSON(steps.commented.outputs.result).java }} - continue-on-error: true - - name: Verify with Maven - if: steps.check.outputs.triggered == 'true' - run: | - mvn --batch-mode --update-snapshots \ - -DclickhouseVersion=${{ fromJSON(steps.commented.outputs.result).clickhouse }} \ - -DclickhouseTimezone=Asia/Chongqing \ - -Duser.timezone=America/Los_Angeles verify - id: maven - continue-on-error: true - - name: Comment PR - uses: actions/github-script@v3 - if: steps.check.outputs.triggered == 'true' - env: - CLICKHOUSE_VERSION: ${{ fromJSON(steps.commented.outputs.result).clickhouse }} - JAVA_VERSION: ${{ fromJSON(steps.commented.outputs.result).java }} - PREV_STEP_RESULT: '${{ steps.maven.outcome }}' - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { issue: { number: issue_number }, repo: { owner, repo } } = context; - const result = process.env.PREV_STEP_RESULT; - const buildUrl = `https://github.com/${owner}/${repo}/actions/runs/${context.runId}`; - const flag = result === 'success' - ? ':green_circle:' - : (result === 'failure' ? ':red_circle:' : ':yellow_circle:'); - const msg = `${flag} verify using JDK [${process.env.JAVA_VERSION}] and ClickHouse [${process.env.CLICKHOUSE_VERSION}]: [${result}](${buildUrl})`; - github.issues.createComment({ issue_number, owner, repo, body: msg }); - - verify-on-demand: - runs-on: ubuntu-latest - name: Verify on demand - if: github.event_name == 'workflow_dispatch' - steps: - - name: Check out repository - uses: actions/checkout@v2 - - name: Check out PR - run: | - git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin pull/${{ github.event.inputs.pr }}/merge:merged-pr && git checkout merged-pr - if: github.event.inputs.pr != '' - - name: Set up JDK ${{ github.event.inputs.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ github.event.inputs.java }} - continue-on-error: true - - name: Verify with Maven - run: | - mvn --batch-mode --update-snapshots \ - -DclickhouseVersion=${{ github.event.inputs.clickhouse }} \ - -DclickhouseTimezone=${{ github.event.inputs.chTz }} \ - -Duser.timezone=${{ github.event.inputs.javaTz }} verify - id: maven - continue-on-error: true - - name: Comment PR - uses: actions/github-script@v3 - if: github.event.inputs.pr != '' - env: - PR_NO: ${{ github.event.inputs.pr }} - CLICKHOUSE_TIMEZONE: ${{ github.event.inputs.chTz }} - CLICKHOUSE_VERSION: ${{ github.event.inputs.clickhouse }} - JAVA_TIMEZONE: ${{ github.event.inputs.javaTz }} - JAVA_VERSION: ${{ github.event.inputs.java }} - PREV_STEP_RESULT: '${{ steps.maven.outcome }}' - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const issue_number = process.env.PR_NO; - const { repo: { owner, repo } } = context; - const result = process.env.PREV_STEP_RESULT; - const buildUrl = `https://github.com/${owner}/${repo}/actions/runs/${context.runId}`; - const flag = result === 'success' - ? ':green_circle:' - : (result === 'failure' ? ':red_circle:' : ':yellow_circle:'); - const msg = `${flag} verify using JDK [${process.env.JAVA_VERSION}, timezone=${process.env.JAVA_TIMEZONE}] and ClickHouse [${process.env.CLICKHOUSE_VERSION}, timezone=${process.env.CLICKHOUSE_TIMEZONE}]: [${result}](${buildUrl})`; - github.issues.createComment({ issue_number, owner, repo, body: msg }); diff --git a/README.md b/README.md index 3c6f4fe01..c06d9cebd 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ try (ClickHouseClient client = ClickHouseClient.newInstance(ClickHouseProtocol.G ClickHouseResponse resp = client.connect(server) .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).set("send_logs_level", "trace") .query("select id, name from some_table where id in :ids and name like :name").params(Arrays.asList(1,2,3), "%key%").execute().get()) { - // you can also use resp.recordStream() as well + // you can also use resp.stream() as well for (ClickHouseRecord record : resp.records()) { int id = record.getValue(0).asInteger(); String name = record.getValue(1).asString(); diff --git a/clickhouse-benchmark/pom.xml b/clickhouse-benchmark/pom.xml index f49918cc0..fe975208c 100644 --- a/clickhouse-benchmark/pom.xml +++ b/clickhouse-benchmark/pom.xml @@ -17,10 +17,7 @@ 1.4.4 - 2.7.3 - 8.0.26 - 2.5.6 - 42.2.23 + 2.6.0 UTF-8 1.33 benchmarks @@ -67,10 +64,10 @@ - ru.yandex.clickhouse + com.clickhouse clickhouse-jdbc ${revision} - shaded + all * @@ -79,9 +76,9 @@ - cc.blynk.clickhouse - clickhouse4j - ${clickhouse4j-driver.version} + com.github.housepower + clickhouse-native-jdbc-shaded + ${native-driver.version} * @@ -90,9 +87,9 @@ - org.mariadb.jdbc - mariadb-java-client - ${mariadb-driver.version} + cc.blynk.clickhouse + clickhouse4j + ${clickhouse4j-driver.version} * @@ -103,35 +100,14 @@ mysql mysql-connector-java - ${mysql-driver.version} - - - * - * - - - com.github.housepower - clickhouse-native-jdbc-shaded - ${native-driver.version} - - - * - * - - + org.mariadb.jdbc + mariadb-java-client org.postgresql postgresql - ${postgresql-driver.version} - - - * - * - - org.testcontainers diff --git a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/client/ClientState.java b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/client/ClientState.java index 499ae7118..4aada4760 100644 --- a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/client/ClientState.java +++ b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/client/ClientState.java @@ -24,7 +24,7 @@ import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseValue; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.grpc.config.ClickHouseGrpcClientOption; +import com.clickhouse.client.grpc.config.ClickHouseGrpcOption; @State(Scope.Thread) public class ClientState extends BaseState { @@ -57,24 +57,25 @@ private ClickHouseClient createClient() { ClickHouseClientBuilder builder = ClickHouseClient.builder(); if (bufferSize != null && !bufferSize.isEmpty()) { - builder.addOption(ClickHouseClientOption.MAX_BUFFER_SIZE, Integer.parseInt(bufferSize)); + builder.option(ClickHouseClientOption.MAX_BUFFER_SIZE, Integer.parseInt(bufferSize)); } if (compression != null && !compression.isEmpty()) { - builder.addOption(ClickHouseClientOption.COMPRESSION, compression.toUpperCase()); + // builder.option(ClickHouseClientOption.COMPRESSION, + // compression.toUpperCase()); if (ClickHouseCompression.NONE.name().equalsIgnoreCase(compression)) { - builder.addOption(ClickHouseGrpcClientOption.USE_FULL_STREAM_DECOMPRESSION, true); + builder.option(ClickHouseGrpcOption.USE_FULL_STREAM_DECOMPRESSION, true); } } if (threads != null && !threads.isEmpty()) { - builder.addOption(ClickHouseClientOption.MAX_THREADS_PER_CLIENT, Integer.parseInt(threads)); + builder.option(ClickHouseClientOption.MAX_THREADS_PER_CLIENT, Integer.parseInt(threads)); } if (window != null && !window.isEmpty()) { - builder.addOption(ClickHouseGrpcClientOption.FLOW_CONTROL_WINDOW, Integer.parseInt(window)); + builder.option(ClickHouseGrpcOption.FLOW_CONTROL_WINDOW, Integer.parseInt(window)); } - return builder.addOption(ClickHouseClientOption.ASYNC, "async".equals(mode)) - .addOption(ClickHouseGrpcClientOption.USE_OKHTTP, "okhttp".equals(transport)).build(); + return builder.option(ClickHouseClientOption.ASYNC, "async".equals(mode)) + .option(ClickHouseGrpcOption.USE_OKHTTP, "okhttp".equals(transport)).build(); } @Setup(Level.Trial) diff --git a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/DriverState.java b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/DriverState.java index 2d7285f55..684124db8 100644 --- a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/DriverState.java +++ b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/DriverState.java @@ -17,8 +17,8 @@ @State(Scope.Thread) public class DriverState extends BaseState { - @Param(value = { "clickhouse4j", "clickhouse-jdbc", "clickhouse-native-jdbc-shaded", "mariadb-java-client", - "mysql-connector-java", "postgresql-jdbc" }) + @Param(value = { "clickhouse4j", "clickhouse-http-jdbc", "clickhouse-grpc-jdbc", "clickhouse-jdbc", + "clickhouse-native-jdbc-shaded", "mariadb-java-client", "mysql-connector-java", "postgresql-jdbc" }) private String client; @Param(value = { Constants.REUSE_CONNECTION, Constants.NEW_CONNECTION }) @@ -38,11 +38,13 @@ public class DriverState extends BaseState { public void doSetup(ServerState serverState) throws Exception { JdbcDriver jdbcDriver = JdbcDriver.from(client); + String compression = String.valueOf(Boolean.parseBoolean(System.getProperty("compression", "true"))); + try { driver = (java.sql.Driver) Class.forName(jdbcDriver.getClassName()).getDeclaredConstructor().newInstance(); url = String.format(jdbcDriver.getUrlTemplate(), serverState.getHost(), serverState.getPort(jdbcDriver.getDefaultPort()), serverState.getDatabase(), serverState.getUser(), - serverState.getPassword()); + serverState.getPassword(), compression); conn = driver.connect(url, new Properties()); try (Statement s = conn.createStatement()) { diff --git a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/JdbcDriver.java b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/JdbcDriver.java index 438f03e18..81adfcb8b 100644 --- a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/JdbcDriver.java +++ b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/JdbcDriver.java @@ -5,32 +5,38 @@ public enum JdbcDriver { // ClickHouse4j Clickhouse4j("cc.blynk.clickhouse.ClickHouseDriver", - "jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC", + "jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s", Constants.HTTP_PORT), // ClickHouse JDBC Driver + ClickhouseHttpJdbc("com.clickhouse.jdbc.ClickHouseDriver", + "jdbc:ch://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s", + Constants.HTTP_PORT), + ClickhouseGrpcJdbc("com.clickhouse.jdbc.ClickHouseDriver", + "jdbc:ch:grpc://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s", + Constants.GRPC_PORT), ClickhouseJdbc("ru.yandex.clickhouse.ClickHouseDriver", - "jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC", + "jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s", Constants.HTTP_PORT), // ClickHouse Native JDBC Driver ClickhouseNativeJdbcShaded("com.github.housepower.jdbc.ClickHouseDriver", - "jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC", + "jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s", Constants.NATIVE_PORT), // MariaDB Java Client MariadbJavaClient("org.mariadb.jdbc.Driver", - "jdbc:mariadb://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false" + "jdbc:mariadb://%s:%s/%s?user=%s&password=%s&useSSL=false&useServerPrepStmts=false&useCompression=%s" + "&rewriteBatchedStatements=true&cachePrepStmts=true&serverTimezone=UTC", Constants.MYSQL_PORT), // MySQL Connector/J MysqlConnectorJava("com.mysql.cj.jdbc.Driver", - "jdbc:mysql://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false" - + "&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC", + "jdbc:mysql://%s:%s/%s?user=%s&password=%s&useSSL=false&useServerPrepStmts=false" + + "&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC&useCompression=%s", Constants.MYSQL_PORT), // PostgreSQL JDBC Driver PostgresqlJdbc("org.postgresql.Driver", - "jdbc:postgresql://%s:%s/%s?user=%s&password=%s&ssl=false&sslmode=disable&preferQueryMode=simple", + "jdbc:postgresql://%s:%s/%s?user=%s&password=%s&ssl=false&sslmode=disable&preferQueryMode=simple&compress=%s", Constants.POSTGRESQL_PORT); private final String className; diff --git a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/Query.java b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/Query.java index b835e2e95..c5148f1fa 100644 --- a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/Query.java +++ b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/Query.java @@ -11,7 +11,7 @@ public void selectDateTime32Rows(Blackhole blackhole, DriverState state) throws int num = state.getRandomNumber(); int rows = state.getSampleSize() + num; try (Statement stmt = executeQuery(state, - "select toDateTime('2021-02-20 13:15:20') + number as d from system.numbers limit ?", rows)) { + "select toDateTime32(1613826920 + number) as d from system.numbers limit ?", rows)) { ResultSet rs = stmt.getResultSet(); while (rs.next()) { blackhole.consume(rs.getTimestamp(1)); @@ -24,8 +24,7 @@ public void selectDateTime64Rows(Blackhole blackhole, DriverState state) throws int num = state.getRandomNumber(); int rows = state.getSampleSize() + num; try (Statement stmt = executeQuery(state, - "select toDateTime64('2021-02-20 13:15:20.000000000', 9) + number as d from system.numbers limit ?", - rows)) { + "select toDateTime64(1613826920 + number / 1000000000, 9) as d from system.numbers limit ?", rows)) { ResultSet rs = stmt.getResultSet(); while (rs.next()) { blackhole.consume(rs.getTimestamp(1)); @@ -33,11 +32,24 @@ public void selectDateTime64Rows(Blackhole blackhole, DriverState state) throws } } + @Benchmark + public void selectDateTime64ObjectRows(Blackhole blackhole, DriverState state) throws Throwable { + int num = state.getRandomNumber(); + int rows = state.getSampleSize() + num; + try (Statement stmt = executeQuery(state, + "select toDateTime64(1613826920 + number / 1000000000, 9) as d from system.numbers limit ?", rows)) { + ResultSet rs = stmt.getResultSet(); + while (rs.next()) { + blackhole.consume(rs.getObject(1)); + } + } + } + @Benchmark public void selectInt32Rows(Blackhole blackhole, DriverState state) throws Throwable { int num = state.getRandomNumber(); int rows = state.getSampleSize() + num; - try (Statement stmt = executeQuery(state, "select * from system.numbers limit ?", rows)) { + try (Statement stmt = executeQuery(state, "select toInt32(number) from system.numbers limit ?", rows)) { ResultSet rs = stmt.getResultSet(); while (rs.next()) { blackhole.consume(rs.getInt(1)); diff --git a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/FactoryBenchmark.java b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/FactoryBenchmark.java index 5f2216fa4..db7c03c10 100644 --- a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/FactoryBenchmark.java +++ b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/FactoryBenchmark.java @@ -43,7 +43,6 @@ import com.clickhouse.client.data.ClickHouseLongValue; import com.clickhouse.client.data.ClickHouseShortValue; import com.clickhouse.client.data.ClickHouseStringValue; -import com.clickhouse.client.data.ClickHouseTimeValue; @State(Scope.Benchmark) @Warmup(iterations = 10, timeUnit = TimeUnit.SECONDS, time = 1) @@ -89,7 +88,7 @@ public void setupSamples() { // add(map, list, Object[].class, () -> ClickHouseArrayValue.of((Object[]) o)); add(map, list, LocalDate.class, () -> ClickHouseDateValue.ofNull()); - add(map, list, LocalTime.class, () -> ClickHouseTimeValue.ofNull()); + add(map, list, LocalTime.class, () -> ClickHouseDateTimeValue.ofNull(0)); add(map, list, LocalDateTime.class, () -> ClickHouseDateTimeValue.ofNull(0)); add(map, list, String.class, () -> ClickHouseStringValue.ofNull()); @@ -130,7 +129,7 @@ ClickHouseValue newValue(Class clazz) { } else if (LocalDate.class.equals(clazz)) { return ClickHouseDateValue.ofNull(); } else if (LocalTime.class.equals(clazz)) { - return ClickHouseTimeValue.ofNull(); + return ClickHouseDateTimeValue.ofNull(0); } else if (LocalDateTime.class.equals(clazz)) { return ClickHouseDateTimeValue.ofNull(0); } else if (String.class.equals(clazz)) { diff --git a/clickhouse-client/README.md b/clickhouse-client/README.md index 565cc3902..08da45f26 100644 --- a/clickhouse-client/README.md +++ b/clickhouse-client/README.md @@ -1,27 +1,22 @@ # ClickHouse Java Client -Async Java client for ClickHouse. `clickhouse-client` is an abstract module, so it does not work by itself unless being used together with implementation module like `ckhouse-grpc-client` or `clickhouse-http-client`. +Async Java client for ClickHouse. `clickhouse-client` is an abstract module, so it does not work by itself until being used together with an implementation like `clickhouse-grpc-client` or `clickhouse-http-client`. ## Quick Start -```java -import java.time.LocalDateTime; -import java.util.concurrent.CompletableFuture; -import java.util.List; - -import com.clickhouse.client.ClickHouseClient; -import com.clickhouse.client.ClickHouseCluster; -import com.clickhouse.client.ClickHouseNode; -import com.clickhouse.client.ClickHouseProtocol; -import com.clickhouse.client.ClickHouseRecord; -import com.clickhouse.client.ClickHouseResponse; -import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.ClickHouseValue; +```xml + + com.clickhouse + clickhouse-http-client + 0.3.2 + +``` +```java // declare a server to connect to -ClickHouseNode server = ClickHouseNode.of("server1.domain", ClickHouseProtocol.GRPC, 9100, "my_db"); +ClickHouseNode server = ClickHouseNode.of("server1.domain", ClickHouseProtocol.HTTP, 8123, "my_db"); -// execute multiple queries one after another within one session +// execute multiple queries in a worker thread one after another within same session CompletableFuture> future = ClickHouseClient.send(server, "create database if not exists test", "use test", // change current database from my_db to test @@ -30,33 +25,32 @@ CompletableFuture> future = ClickHouseClient.sen "select * from test_table limit 1", "truncate table test_table", "drop table if exists test_table"); -// do something else in current thread, and then retrieve summaries -List results = future.get(); - -// declare a cluster -ClickHouseCluster cluster = ClickHouseCluster.builder() - // defaults to localhost:8123 and http protocol - .addNode(ClickHouseNode.builder().cluster("cluster1").tags("dc1", "rack1", "for-write").build()) - .addNode(ClickHouseNode.of("1.2.3.4", ClickHouseProtocol.GRPC, 9100, "system", "dc2", "rack2", "for-read")) - .build(); - -// issue query against one node via grpc -String sql = "select * from numbers(100)"; -try (ClickHouseClient client = ClickHouseClient.newInstance(ClickHouseProtocol.GRPC); - // connect to a node which understands gRPC - ClickHouseResponse response = client.connect(cluster).query(sql).execute().get()) { - for (ClickHouseRecord record : response.records()) { - // Don't cache ClickHouseValue as it's a container object reused among all records - ClickHouseValue v = record.getValue(0); - // converts to DateTime64(6) - LocalDateTime dateTime = v.asDateTime(6); - // converts to long/int/byte if you want to - long l = v.asLong(); - int i = v.asInteger(); - byte b = v.asByte(); - } - // summary will be fully available after all records being retrieved - ClickHouseResponseSummary summary = response.getSummary(); +// block current thread until queries completed, and then retrieve summaries +// List results = future.get(); + +try (ClickHouseClient client = ClickHouseClient.newInstance(server.getProtocol)) { + ClickHouseRequest request = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes); + // load data into a table and wait until it's completed + request.write().query("insert into my_table select c2, c3 from input('c1 UInt8, c2 String, c3 Int32')") + .data(myInputStream).execute().thenAccept(response -> { + response.close(); + }); + + // query with named parameter + try (ClickHouseResponse response = request.query(ClickHouseParameterizedQuery.of( + "select * from numbers(:limit)")).params(100000).execute().get()) { + for (ClickHouseRecord r : response.records()) { + // Don't cache ClickHouseValue / ClickHouseRecord as they're reused for + // corresponding column / row + ClickHouseValue v = r.getValue(0); + // converts to DateTime64(6) + LocalDateTime dateTime = v.asDateTime(6); + // converts to long/int/byte if you want to + long l = v.asLong(); + int i = v.asInteger(); + byte b = v.asByte(); + } + } } ``` diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/AbstractClient.java b/clickhouse-client/src/main/java/com/clickhouse/client/AbstractClient.java new file mode 100644 index 000000000..aa95a8f4e --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/AbstractClient.java @@ -0,0 +1,239 @@ +package com.clickhouse.client; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import com.clickhouse.client.logging.Logger; +import com.clickhouse.client.logging.LoggerFactory; + +/** + * Base class for implementing a thread-safe ClickHouse client. It uses + * {@link ReadWriteLock} to manage access to underlying connection. + */ +public abstract class AbstractClient implements ClickHouseClient { + private static final Logger log = LoggerFactory.getLogger(AbstractClient.class); + + private boolean initialized = false; + + private ExecutorService executor = null; + private ClickHouseConfig config = null; + private ClickHouseNode server = null; + private T connection = null; + + protected final ReadWriteLock lock = new ReentrantReadWriteLock(); + + private void ensureInitialized() { + if (!initialized) { + throw new IllegalStateException("Please initialize the client first"); + } + } + + // just for testing purpose + final boolean isInitialized() { + return initialized; + } + + protected CompletableFuture failedResponse(Throwable ex) { + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(ex); + return future; + } + + /** + * Gets executor service for this client. + * + * @return executor service + * @throws IllegalStateException when the client is either closed or not + * initialized + */ + protected final ExecutorService getExecutor() { + lock.readLock().lock(); + try { + ensureInitialized(); + return executor; + } finally { + lock.readLock().unlock(); + } + } + + /** + * Gets current server. + * + * @return current server + * @throws IllegalStateException when the client is either closed or not + * initialized + */ + protected final ClickHouseNode getServer() { + lock.readLock().lock(); + try { + ensureInitialized(); + return server; + } finally { + lock.readLock().unlock(); + } + } + + /** + * Checks if the underlying connection can be reused. In general, new connection + * will be created when {@code connection} is null or {@code requestServer} is + * different from {@code currentServer} - the existing connection will be closed + * in the later case. + * + * @param connection existing connection which may or may not be null + * @param requestServer non-null requested server, returned from previous call + * of {@code request.getServer()} + * @param currentServer current server, same as {@code getServer()} + * @param request non-null request + * @return true if the connection should NOT be changed(e.g. requestServer is + * same as currentServer); false otherwise + */ + protected boolean checkConnection(T connection, ClickHouseNode requestServer, ClickHouseNode currentServer, + ClickHouseRequest request) { + return connection != null && requestServer.equals(currentServer); + } + + /** + * Creates a new connection and optionally close existing connection. This + * method will be called from {@link #getConnection(ClickHouseRequest)} as + * needed. + * + * @param connection existing connection which may or may not be null + * @param server non-null requested server, returned from previous call of + * {@code request.getServer()} + * @param request non-null request + * @return new connection + * @throws CompletionException when error occured + */ + protected abstract T newConnection(T connection, ClickHouseNode server, ClickHouseRequest request); + + /** + * Closes a connection. This method will be called from {@link #close()}. + * + * @param connection connection to close + * @param force whether force to close the connection or not + */ + protected abstract void closeConnection(T connection, boolean force); + + /** + * Gets a connection according to the given request. + * + * @param request non-null request + * @return non-null connection + * @throws CompletionException when error occured + */ + protected final T getConnection(ClickHouseRequest request) { + ClickHouseNode newNode = ClickHouseChecker.nonNull(request, "request").getServer(); + lock.readLock().lock(); + try { + ensureInitialized(); + if (checkConnection(connection, newNode, server, request)) { + return connection; + } + } finally { + lock.readLock().unlock(); + } + + lock.writeLock().lock(); + try { + server = newNode; + log.debug("Connecting to: %s", newNode); + connection = newConnection(connection, server, request); + log.debug("Connection established: %s", connection); + + return connection; + } finally { + lock.writeLock().unlock(); + } + } + + @Override + public final ClickHouseConfig getConfig() { + lock.readLock().lock(); + try { + ensureInitialized(); + return config; + } finally { + lock.readLock().unlock(); + } + } + + @Override + public void init(ClickHouseConfig config) { + ClickHouseChecker.nonNull(config, "config"); + + lock.writeLock().lock(); + try { + this.config = config; + if (this.executor == null) { // only initialize once + int threads = config.getMaxThreadsPerClient(); + this.executor = threads <= 0 ? ClickHouseClient.getExecutorService() + : ClickHouseUtils.newThreadPool(getClass().getSimpleName(), threads, + config.getMaxQueuedRequests()); + } + + initialized = true; + } finally { + lock.writeLock().unlock(); + } + } + + @Override + public final void close() { + lock.readLock().lock(); + try { + if (!initialized) { + return; + } + } finally { + lock.readLock().unlock(); + } + + lock.writeLock().lock(); + try { + server = null; + + if (executor != null) { + executor.shutdown(); + } + + if (connection != null) { + closeConnection(connection, false); + } + + // shutdown* won't shutdown commonPool, so awaitTermination will always time out + // on the other hand, for a client-specific thread pool, we'd better shut it + // down for real + if (executor != null && config.getMaxThreadsPerClient() > 0 + && !executor.awaitTermination(config.getConnectionTimeout(), TimeUnit.MILLISECONDS)) { + executor.shutdownNow(); + } + + executor = null; + connection = null; + } catch (InterruptedException e) { + log.warn("Got interrupted when closing client", e); + Thread.currentThread().interrupt(); + } catch (Exception e) { + log.warn("Exception occurred when closing client", e); + } finally { + initialized = false; + try { + if (connection != null) { + closeConnection(connection, true); + } + + if (executor != null) { + executor.shutdownNow(); + } + } finally { + executor = null; + connection = null; + lock.writeLock().unlock(); + } + } + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseAggregateFunction.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseAggregateFunction.java new file mode 100644 index 000000000..1f5273135 --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseAggregateFunction.java @@ -0,0 +1,114 @@ +package com.clickhouse.client; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +public enum ClickHouseAggregateFunction { + // select concat(f.name, '(', f.case_insensitive ? 'true' : 'false', + // a.names != '' ? concat(',', replaceAll(replaceRegexpAll(a.names, + // '^\\[(.*)\\]$', '\\1'), '''', '"')) : '', '),') as x + // from system.functions f + // left outer join ( + // select alias_to, toString(groupArray(case_insensitive ? upper(name) : name)) + // as names + // from system.functions + // where is_aggregate != 0 and alias_to != '' + // group by alias_to + // ) a on f.name = a.alias_to + // where f.is_aggregate != 0 and f.alias_to = '' + // order by f.name + aggThrow(false), any(false), anyHeavy(false), anyLast(false), argMax(false), argMin(false), avg(true), + avgWeighted(false), boundingRatio(false), categoricalInformationValue(false), corr(true), corrStable(false), + count(true), covarPop(false, "COVAR_POP"), covarPopStable(false), covarSamp(false, "COVAR_SAMP"), + covarSampStable(false), deltaSum(false), deltaSumTimestamp(false), dense_rank(false), entropy(false), + first_value(true), groupArray(false), groupArrayInsertAt(false), groupArrayMovingAvg(false), + groupArrayMovingSum(false), groupArraySample(false), groupBitAnd(false, "BIT_AND"), groupBitOr(false, "BIT_OR"), + groupBitXor(false, "BIT_XOR"), groupBitmap(false), groupBitmapAnd(false), groupBitmapOr(false), + groupBitmapXor(false), groupUniqArray(false), histogram(false), intervalLengthSum(false), kurtPop(false), + kurtSamp(false), lagInFrame(false), last_value(true), leadInFrame(false), mannWhitneyUTest(false), max(true), + maxIntersections(false), maxIntersectionsPosition(false), maxMap(false), min(true), minMap(false), + quantile(false, "median"), quantileBFloat16(false, "medianBFloat16"), + quantileDeterministic(false, "medianDeterministic"), quantileExact(false, "medianExact"), + quantileExactExclusive(false), quantileExactHigh(false, "medianExactHigh"), quantileExactInclusive(false), + quantileExactLow(false, "medianExactLow"), quantileExactWeighted(false, "medianExactWeighted"), + quantileTDigest(false, "medianTDigest"), quantileTDigestWeighted(false, "medianTDigestWeighted"), + quantileTiming(false, "medianTiming"), quantileTimingWeighted(false, "medianTimingWeighted"), quantiles(false), + quantilesBFloat16(false), quantilesDeterministic(false), quantilesExact(false), quantilesExactExclusive(false), + quantilesExactHigh(false), quantilesExactInclusive(false), quantilesExactLow(false), quantilesExactWeighted(false), + quantilesTDigest(false), quantilesTDigestWeighted(false), quantilesTiming(false), quantilesTimingWeighted(false), + rank(false), rankCorr(false), retention(false), row_number(false), sequenceCount(false), sequenceMatch(false), + sequenceNextNode(false), simpleLinearRegression(false), skewPop(false), skewSamp(false), + stddevPop(false, "STDDEV_POP"), stddevPopStable(false), stddevSamp(false, "STDDEV_SAMP"), stddevSampStable(false), + stochasticLinearRegression(false), stochasticLogisticRegression(false), studentTTest(false), sum(true), + sumCount(false), sumKahan(false), sumMap(false), sumMapFiltered(false), sumMapFilteredWithOverflow(false), + sumMapWithOverflow(false), sumWithOverflow(false), topK(false), topKWeighted(false), uniq(false), + uniqCombined(false), uniqCombined64(false), uniqExact(false), uniqHLL12(false), uniqTheta(false), uniqUpTo(false), + varPop(false, "VAR_POP"), varPopStable(false), varSamp(false, "VAR_SAMP"), varSampStable(false), welchTTest(false), + windowFunnel(false); + + public static final Map name2func; + + static { + Map map = new HashMap<>(); + String errorMsg = "[%s] is used by type [%s]"; + ClickHouseAggregateFunction used = null; + for (ClickHouseAggregateFunction t : ClickHouseAggregateFunction.values()) { + String name = t.name(); + if (!t.isCaseSensitive()) { + name = name.toUpperCase(); + } + used = map.put(name, t); + if (used != null) { + throw new IllegalStateException(String.format(Locale.ROOT, errorMsg, name, used.name())); + } + } + + name2func = Collections.unmodifiableMap(map); + } + + /** + * Converts given type name to corresponding aggregate function. + * + * @param function non-empty function + * @return aggregate function + */ + public static ClickHouseAggregateFunction of(String function) { + if (function == null || (function = function.trim()).isEmpty()) { + throw new IllegalArgumentException("Non-empty function is required"); + } + + ClickHouseAggregateFunction f = name2func.get(function); + if (f == null) { + f = name2func.get(function.toUpperCase()); // case-insensitive or just an alias + } + + if (f == null) { + throw new IllegalArgumentException("Unknown aggregate function: " + function); + } + return f; + } + + private final boolean caseSensitive; + private final List aliases; + + ClickHouseAggregateFunction(boolean caseSensitive, String... aliases) { + this.caseSensitive = caseSensitive; + if (aliases == null || aliases.length == 0) { + this.aliases = Collections.emptyList(); + } else { + this.aliases = Collections.unmodifiableList(Arrays.asList(aliases)); + } + } + + public boolean isCaseSensitive() { + return caseSensitive; + } + + public List getAliases() { + return aliases; + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java index ce5cc626d..74c98bc4c 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; @@ -18,9 +19,8 @@ import java.util.function.Function; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseDefaults; +import com.clickhouse.client.config.ClickHouseOption; import com.clickhouse.client.data.ClickHousePipedStream; -import com.clickhouse.client.exception.ClickHouseException; -import com.clickhouse.client.exception.ClickHouseExceptionSpecifier; /** * A unified interface defines Java client for ClickHouse. A client can only @@ -76,14 +76,28 @@ static CompletableFuture submit(Callable task) { return (boolean) ClickHouseDefaults.ASYNC.getEffectiveDefaultValue() ? CompletableFuture.supplyAsync(() -> { try { return task.call(); + } catch (CompletionException e) { + throw e; } catch (Exception e) { - throw new CompletionException(e); + Throwable cause = e.getCause(); + if (cause instanceof CompletionException) { + throw (CompletionException) cause; + } else if (cause == null) { + cause = e; + } + throw new CompletionException(cause); } }, getExecutorService()) : CompletableFuture.completedFuture(task.call()); } catch (CompletionException e) { throw e; } catch (Exception e) { - throw new CompletionException(e); + Throwable cause = e.getCause(); + if (cause instanceof CompletionException) { + throw (CompletionException) cause; + } else if (cause == null) { + cause = e; + } + throw new CompletionException(cause); } } @@ -99,6 +113,7 @@ static CompletableFuture submit(Callable task) { * @return future object to get result * @throws IllegalArgumentException if any of server, tableOrQuery, and output * is null + * @throws CompletionException when error occurred during execution * @throws IOException when failed to create the file or its parent * directories */ @@ -121,6 +136,7 @@ static CompletableFuture dump(ClickHouseNode server, * @return future object to get result * @throws IllegalArgumentException if any of server, tableOrQuery, and output * is null + * @throws CompletionException when error occurred during execution */ static CompletableFuture dump(ClickHouseNode server, String tableOrQuery, ClickHouseFormat format, ClickHouseCompression compression, OutputStream output) { @@ -132,12 +148,11 @@ static CompletableFuture dump(ClickHouseNode server, final ClickHouseNode theServer = ClickHouseCluster.probe(server); final String theQuery = tableOrQuery.trim(); - final ClickHouseFormat theFormat = format == null ? ClickHouseFormat.TabSeparated : format; - final ClickHouseCompression theCompression = compression == null ? ClickHouseCompression.NONE : compression; return submit(() -> { try (ClickHouseClient client = newInstance(theServer.getProtocol())) { - ClickHouseRequest request = client.connect(theServer).compression(theCompression).format(theFormat); + ClickHouseRequest request = client.connect(theServer).compressServerResponse( + compression != null && compression != ClickHouseCompression.NONE, compression).format(format); // FIXME what if the table name is `try me`? if (theQuery.indexOf(' ') < 0) { request.table(theQuery); @@ -146,14 +161,16 @@ static CompletableFuture dump(ClickHouseNode server, } try (ClickHouseResponse response = request.execute().get()) { - response.dump(output); + response.pipe(output, 8192); return response.getSummary(); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw ClickHouseExceptionSpecifier.specify(e, theServer); + throw ClickHouseException.forCancellation(e, theServer); + } catch (CancellationException e) { + throw ClickHouseException.forCancellation(e, theServer); } catch (ExecutionException e) { - throw ClickHouseExceptionSpecifier.handle(e, theServer); + throw ClickHouseException.of(e, theServer); } finally { try { output.close(); @@ -175,6 +192,7 @@ static CompletableFuture dump(ClickHouseNode server, * @param file file to load * @return future object to get result * @throws IllegalArgumentException if any of server, table, and input is null + * @throws CompletionException when error occurred during execution * @throws FileNotFoundException when file not found */ static CompletableFuture load(ClickHouseNode server, String table, @@ -193,6 +211,7 @@ static CompletableFuture load(ClickHouseNode server, * @param writer non-null custom writer to generate data * @return future object to get result * @throws IllegalArgumentException if any of server, table, and writer is null + * @throws CompletionException when error occurred during execution */ static CompletableFuture load(ClickHouseNode server, String table, ClickHouseFormat format, ClickHouseCompression compression, ClickHouseWriter writer) { @@ -203,20 +222,19 @@ static CompletableFuture load(ClickHouseNode server, // in case the protocol is ANY final ClickHouseNode theServer = ClickHouseCluster.probe(server); - final ClickHouseFormat theFormat = format == null ? ClickHouseFormat.TabSeparated : format; - final ClickHouseCompression theCompression = compression == null ? ClickHouseCompression.NONE : compression; - return submit(() -> { InputStream input = null; // must run in async mode so that we won't hold everything in memory try (ClickHouseClient client = ClickHouseClient.builder() .nodeSelector(ClickHouseNodeSelector.of(theServer.getProtocol())) - .addOption(ClickHouseClientOption.ASYNC, true).build()) { + .option(ClickHouseClientOption.ASYNC, true).build()) { ClickHousePipedStream stream = ClickHouseDataStreamFactory.getInstance() .createPipedStream(client.getConfig()); // execute query in a separate thread(because async is explicitly set to true) CompletableFuture future = client.connect(theServer).write().table(table) - .compression(theCompression).format(theFormat).data(input = stream.getInput()).execute(); + .decompressClientRequest(compression != null && compression != ClickHouseCompression.NONE, + compression) + .format(format).data(input = stream.getInput()).execute(); try { // write data into stream in current thread writer.write(stream); @@ -229,9 +247,11 @@ static CompletableFuture load(ClickHouseNode server, } } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw ClickHouseExceptionSpecifier.specify(e, theServer); + throw ClickHouseException.forCancellation(e, theServer); + } catch (CancellationException e) { + throw ClickHouseException.forCancellation(e, theServer); } catch (ExecutionException e) { - throw ClickHouseExceptionSpecifier.handle(e, theServer); + throw ClickHouseException.of(e, theServer); } finally { if (input != null) { try { @@ -256,6 +276,7 @@ static CompletableFuture load(ClickHouseNode server, * end of the call * @return future object to get result * @throws IllegalArgumentException if any of server, table, and input is null + * @throws CompletionException when error occurred during execution */ static CompletableFuture load(ClickHouseNode server, String table, ClickHouseFormat format, ClickHouseCompression compression, InputStream input) { @@ -266,19 +287,20 @@ static CompletableFuture load(ClickHouseNode server, // in case the protocol is ANY final ClickHouseNode theServer = ClickHouseCluster.probe(server); - final ClickHouseFormat theFormat = format == null ? ClickHouseFormat.TabSeparated : format; - final ClickHouseCompression theCompression = compression == null ? ClickHouseCompression.NONE : compression; - return submit(() -> { try (ClickHouseClient client = newInstance(theServer.getProtocol()); ClickHouseResponse response = client.connect(theServer).write().table(table) - .compression(theCompression).format(theFormat).data(input).execute().get()) { + .decompressClientRequest(compression != null && compression != ClickHouseCompression.NONE, + compression) + .format(format).data(input).execute().get()) { return response.getSummary(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw ClickHouseExceptionSpecifier.specify(e, theServer); + throw ClickHouseException.forCancellation(e, theServer); + } catch (CancellationException e) { + throw ClickHouseException.forCancellation(e, theServer); } catch (ExecutionException e) { - throw ClickHouseExceptionSpecifier.handle(e, theServer); + throw ClickHouseException.of(e, theServer); } finally { try { input.close(); @@ -309,6 +331,7 @@ static ClickHouseClient newInstance(ClickHouseProtocol... preferredProtocols) { * @param more more SQL queries if any * @return list of {@link ClickHouseResponseSummary} one for each execution * @throws IllegalArgumentException if server or sql is null + * @throws CompletionException when error occurred during execution */ static CompletableFuture> send(ClickHouseNode server, String sql, String... more) { if (server == null || sql == null) { @@ -333,7 +356,7 @@ static CompletableFuture> send(ClickHouseNode se // set async to false so that we don't have to create additional thread try (ClickHouseClient client = ClickHouseClient.builder() .nodeSelector(ClickHouseNodeSelector.of(theServer.getProtocol())) - .addOption(ClickHouseClientOption.ASYNC, false).build()) { + .option(ClickHouseClientOption.ASYNC, false).build()) { ClickHouseRequest request = client.connect(theServer).format(ClickHouseFormat.RowBinary); if ((boolean) ClickHouseDefaults.AUTO_SESSION.getEffectiveDefaultValue() && queries.size() > 1) { request.session(UUID.randomUUID().toString(), false); @@ -343,6 +366,13 @@ static CompletableFuture> send(ClickHouseNode se list.add(resp.getSummary()); } } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw ClickHouseException.forCancellation(e, theServer); + } catch (CancellationException e) { + throw ClickHouseException.forCancellation(e, theServer); + } catch (ExecutionException e) { + throw ClickHouseException.of(e, theServer); } return list; @@ -357,6 +387,7 @@ static CompletableFuture> send(ClickHouseNode se * @param params non-null stringified parameters * @return list of {@link ClickHouseResponseSummary} one for each execution * @throws IllegalArgumentException if any of server, sql, and params is null + * @throws CompletionException when error occurred during execution */ static CompletableFuture send(ClickHouseNode server, String sql, Map params) { @@ -371,10 +402,17 @@ static CompletableFuture send(ClickHouseNode server, // set async to false so that we don't have to create additional thread try (ClickHouseClient client = ClickHouseClient.builder() .nodeSelector(ClickHouseNodeSelector.of(theServer.getProtocol())) - .addOption(ClickHouseClientOption.ASYNC, false).build(); + .option(ClickHouseClientOption.ASYNC, false).build(); ClickHouseResponse resp = client.connect(theServer).format(ClickHouseFormat.RowBinary).query(sql) .params(params).execute().get()) { return resp.getSummary(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw ClickHouseException.forCancellation(e, theServer); + } catch (CancellationException e) { + throw ClickHouseException.forCancellation(e, theServer); + } catch (ExecutionException e) { + throw ClickHouseException.of(e, theServer); } }); } @@ -390,6 +428,7 @@ static CompletableFuture send(ClickHouseNode server, * @return list of {@link ClickHouseResponseSummary} one for each execution * @throws IllegalArgumentException if columns is null, empty or contains null * column + * @throws CompletionException when error occurred during execution */ static CompletableFuture> send(ClickHouseNode server, String sql, List columns, Object[]... params) { @@ -418,6 +457,7 @@ static CompletableFuture> send(ClickHouseNode se * @return list of {@link ClickHouseResponseSummary} one for each execution * @throws IllegalArgumentException if no named parameter in the query, or * templates or params is null or empty + * @throws CompletionException when error occurred during execution */ static CompletableFuture> send(ClickHouseNode server, String sql, ClickHouseValue[] templates, Object[]... params) { @@ -441,7 +481,7 @@ static CompletableFuture> send(ClickHouseNode se // set async to false so that we don't have to create additional thread try (ClickHouseClient client = ClickHouseClient.builder() .nodeSelector(ClickHouseNodeSelector.of(theServer.getProtocol())) - .addOption(ClickHouseClientOption.ASYNC, false).build()) { + .option(ClickHouseClientOption.ASYNC, false).build()) { // format doesn't matter here as we only need a summary ClickHouseRequest request = client.connect(theServer).format(ClickHouseFormat.RowBinary) .query(query); @@ -461,6 +501,13 @@ static CompletableFuture> send(ClickHouseNode se list.add(resp.getSummary()); } } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw ClickHouseException.forCancellation(e, theServer); + } catch (CancellationException e) { + throw ClickHouseException.forCancellation(e, theServer); + } catch (ExecutionException e) { + throw ClickHouseException.of(e, theServer); } return list; @@ -476,6 +523,7 @@ static CompletableFuture> send(ClickHouseNode se * @return list of {@link ClickHouseResponseSummary} one for each execution * @throws IllegalArgumentException if any of server, sql, or params is null; or * no named parameter in the query + * @throws CompletionException when error occurred during execution */ static CompletableFuture> send(ClickHouseNode server, String sql, String[][] params) { @@ -499,7 +547,7 @@ static CompletableFuture> send(ClickHouseNode se // set async to false so that we don't have to create additional thread try (ClickHouseClient client = ClickHouseClient.builder() .nodeSelector(ClickHouseNodeSelector.of(theServer.getProtocol())) - .addOption(ClickHouseClientOption.ASYNC, false).build()) { + .option(ClickHouseClientOption.ASYNC, false).build()) { // format doesn't matter here as we only need a summary ClickHouseRequest request = client.connect(theServer).format(ClickHouseFormat.RowBinary); for (String[] p : params) { @@ -507,44 +555,19 @@ static CompletableFuture> send(ClickHouseNode se list.add(resp.getSummary()); } } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw ClickHouseException.forCancellation(e, theServer); + } catch (CancellationException e) { + throw ClickHouseException.forCancellation(e, theServer); + } catch (ExecutionException e) { + throw ClickHouseException.of(e, theServer); } return list; }); } - /** - * Tests if the given server is alive or not. Unlike other methods, it's a - * synchronous call with minimum overhead(e.g. tiny buffer, no compression and - * no deserialization etc). - * - * @param server server to test - * @param timeout timeout in millisecond - * @return true if the server is alive; false otherwise - */ - static boolean test(ClickHouseNode server, int timeout) { - if (server != null) { - server = ClickHouseCluster.probe(server, timeout); - - try (ClickHouseClient client = ClickHouseClient.builder() - .nodeSelector(ClickHouseNodeSelector.of(server.getProtocol())) - .addOption(ClickHouseClientOption.ASYNC, false) // use current thread - .addOption(ClickHouseClientOption.CONNECTION_TIMEOUT, timeout) - .addOption(ClickHouseClientOption.SOCKET_TIMEOUT, timeout) - .addOption(ClickHouseClientOption.MAX_BUFFER_SIZE, 8) // actually 4 bytes should be enough - .addOption(ClickHouseClientOption.MAX_QUEUED_BUFFERS, 1).build(); - ClickHouseResponse resp = client.connect(server).compression(ClickHouseCompression.NONE) - .format(ClickHouseFormat.TabSeparated).query("SELECT 1").execute() - .get(timeout, TimeUnit.MILLISECONDS)) { - return true; - } catch (Exception e) { - // ignore - } - } - - return false; - } - /** * Tests whether the given protocol is supported or not. An advanced client can * support as many protocols as needed. @@ -588,9 +611,9 @@ default ClickHouseRequest connect(Function execute(ClickHouseRequest request) throws ClickHouseException; + CompletableFuture execute(ClickHouseRequest request); /** * Gets the immutable configuration associated with this client. In most cases @@ -601,6 +624,15 @@ default ClickHouseRequest connect(Function getOptionClass() { + return null; + } + /** * Initializes the client using immutable configuration extracted from the * builder using {@link ClickHouseClientBuilder#getConfig()}. In general, it's @@ -618,6 +650,36 @@ default void init(ClickHouseConfig config) { ClickHouseChecker.nonNull(config, "configuration"); } + /** + * Tests if the given server is alive or not. Pay attention that it's a + * synchronous call with minimum overhead(e.g. tiny buffer, no compression and + * no deserialization etc). + * + * @param server server to test + * @param timeout timeout in millisecond + * @return true if the server is alive; false otherwise + */ + default boolean ping(ClickHouseNode server, int timeout) { + if (server != null) { + server = ClickHouseCluster.probe(server, timeout); + + try (ClickHouseResponse resp = connect(server) // create request + .option(ClickHouseClientOption.ASYNC, false) // use current thread + .option(ClickHouseClientOption.CONNECTION_TIMEOUT, timeout) + .option(ClickHouseClientOption.SOCKET_TIMEOUT, timeout) + .option(ClickHouseClientOption.MAX_BUFFER_SIZE, 8) // actually 4 bytes should be enough + .option(ClickHouseClientOption.MAX_QUEUED_BUFFERS, 1) // enough with only one buffer + .format(ClickHouseFormat.TabSeparated).query("SELECT 1").execute() + .get(timeout, TimeUnit.MILLISECONDS)) { + return true; + } catch (Exception e) { + // ignore + } + } + + return false; + } + @Override void close(); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClientBuilder.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClientBuilder.java index 6fecf1ed7..df32d81b8 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClientBuilder.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClientBuilder.java @@ -1,13 +1,17 @@ package com.clickhouse.client; +import java.io.Serializable; import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.Properties; import java.util.ServiceLoader; +import java.util.Map.Entry; import java.util.concurrent.ExecutorService; import java.util.concurrent.ForkJoinPool; -import com.clickhouse.client.config.ClickHouseConfigOption; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.config.ClickHouseOption; import com.clickhouse.client.config.ClickHouseDefaults; /** @@ -47,7 +51,7 @@ public class ClickHouseClientBuilder { protected Object metricRegistry; protected ClickHouseNodeSelector nodeSelector; - protected final Map options; + protected final Map options; /** * Default constructor. @@ -106,7 +110,7 @@ public ClickHouseClient build() { if (client == null) { throw new IllegalStateException( - ClickHouseUtils.format("No suitable client(out of %d) found in classpath.", counter)); + ClickHouseUtils.format("No suitable ClickHouse client(out of %d) found in classpath.", counter)); } else { client.init(getConfig()); } @@ -114,17 +118,37 @@ public ClickHouseClient build() { return client; } + /** + * Sets configuration. + * + * @param config non-null configuration + * @return this builder + */ + public ClickHouseClientBuilder config(ClickHouseConfig config) { + this.config = config; + + this.credentials = config.getDefaultCredentials(); + this.metricRegistry = config.getMetricRegistry().orElse(null); + this.nodeSelector = config.getNodeSelector(); + + this.options.putAll(config.getAllOptions()); + + return this; + } + /** * Adds an option, which is usually an Enum type that implements - * {@link com.clickhouse.client.config.ClickHouseConfigOption}. + * {@link com.clickhouse.client.config.ClickHouseOption}. * * @param option non-null option * @param value value * @return this builder */ - public ClickHouseClientBuilder addOption(ClickHouseConfigOption option, Object value) { - Object oldValue = options.put(ClickHouseChecker.nonNull(option, "option"), - ClickHouseChecker.nonNull(value, "value")); + public ClickHouseClientBuilder option(ClickHouseOption option, Serializable value) { + if (option == null || value == null) { + throw new IllegalArgumentException("Non-null option and value are required"); + } + Object oldValue = options.put(option, value); if (oldValue == null || !value.equals(oldValue)) { resetConfig(); } @@ -138,7 +162,7 @@ public ClickHouseClientBuilder addOption(ClickHouseConfigOption option, Object v * @param option non-null option * @return this builder */ - public ClickHouseClientBuilder removeOption(ClickHouseConfigOption option) { + public ClickHouseClientBuilder removeOption(ClickHouseOption option) { Object value = options.remove(ClickHouseChecker.nonNull(option, "option")); if (value != null) { resetConfig(); @@ -150,12 +174,39 @@ public ClickHouseClientBuilder removeOption(ClickHouseConfigOption option) { /** * Sets options. * - * @param options non-null map containing all options + * @param options map containing all options + * @return this builder + */ + public ClickHouseClientBuilder options(Map options) { + if (options != null && !options.isEmpty()) { + this.options.putAll(options); + resetConfig(); + } + + return this; + } + + /** + * Sets options. + * + * @param options options * @return this builder */ - public ClickHouseClientBuilder options(Map options) { - if (ClickHouseChecker.nonNull(options, "options").size() > 0) { - options.putAll(options); + public ClickHouseClientBuilder options(Properties options) { + if (options != null && !options.isEmpty()) { + for (Entry e : options.entrySet()) { + Object key = e.getKey(); + Object value = e.getValue(); + if (key == null || value == null) { + continue; + } + + ClickHouseClientOption o = ClickHouseClientOption.fromKey(key.toString()); + if (o != null) { + this.options.put(o, ClickHouseOption.fromString(value.toString(), o.getValueType())); + } + } + resetConfig(); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java index 851c151ec..b7008471d 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java @@ -185,7 +185,7 @@ public static ClickHouseNode probe(ClickHouseNode node, int timeout) { } else if (buf[3] == 0) { p = ClickHouseProtocol.MYSQL; } else if (buf[0] == 72 && buf[9] == 52) { - p = ClickHouseProtocol.NATIVE; + p = ClickHouseProtocol.TCP; } } } catch (IOException e) { @@ -313,9 +313,18 @@ protected void check() { // detect flaky node and check it in a different way(less frequency) try { boolean passed = true; + int timeout = 5000; for (int i = 0; i < unhealthyNodes.size(); i++) { - ClickHouseNode node = unhealthyNodes.get(i); - if (ClickHouseClient.test(node, 5000)) { // another configuration? + ClickHouseNode node = probe(unhealthyNodes.get(i), timeout); + + // probe is faster than ping but it cannot tell if the server works or not + boolean isAlive = false; + try (ClickHouseClient client = ClickHouseClient.newInstance(node.getProtocol())) { + isAlive = client.ping(node, timeout); + } catch (Exception e) { + // ignore + } + if (isAlive) { // another configuration? update(node, Status.HEALTHY); } else { passed = false; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseColumn.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseColumn.java index 132a276a7..72baac450 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseColumn.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseColumn.java @@ -6,6 +6,7 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Objects; import java.util.TimeZone; /** @@ -26,69 +27,78 @@ public final class ClickHouseColumn implements Serializable { private String originalTypeName; private String columnName; + private ClickHouseAggregateFunction aggFuncType; private ClickHouseDataType dataType; private boolean nullable; private boolean lowCardinality; - private ClickHouseDataType baseType; private TimeZone timeZone; private int precision; private int scale; private List nested; private List parameters; + private int arrayLevel; + private ClickHouseColumn arrayBaseColumn; + private static ClickHouseColumn update(ClickHouseColumn column) { int size = column.parameters.size(); switch (column.dataType) { - case AggregateFunction: - column.baseType = ClickHouseDataType.String; - if (size == 2) { - column.baseType = ClickHouseDataType.of(column.parameters.get(1)); - } - break; - case DateTime: - if (size >= 2) { // same as DateTime64 - column.scale = Integer.parseInt(column.parameters.get(0)); - column.timeZone = TimeZone.getTimeZone(column.parameters.get(1).replace("'", "")); - } else if (size == 1) { // same as DateTime32 - // unfortunately this will fall back to GMT if the time zone - // cannot be resolved - TimeZone tz = TimeZone.getTimeZone(column.parameters.get(0).replace("'", "")); - column.timeZone = tz; - } - break; - case DateTime32: - if (size > 0) { - // unfortunately this will fall back to GMT if the time zone - // cannot be resolved - TimeZone tz = TimeZone.getTimeZone(column.parameters.get(0).replace("'", "")); - column.timeZone = tz; - } - break; - case DateTime64: - if (size > 0) { - column.scale = Integer.parseInt(column.parameters.get(0)); - } - if (size > 1) { - column.timeZone = TimeZone.getTimeZone(column.parameters.get(1).replace("'", "")); - } - break; - case Decimal: - if (size >= 2) { - column.precision = Integer.parseInt(column.parameters.get(0)); - column.scale = Integer.parseInt(column.parameters.get(1)); + case Array: + column.arrayLevel = 1; + column.arrayBaseColumn = column.nested.get(0); + while (column.arrayLevel < 255) { + if (column.arrayBaseColumn.dataType == ClickHouseDataType.Array) { + column.arrayLevel++; + column.arrayBaseColumn = column.arrayBaseColumn.nested.get(0); + } else { + break; } - break; - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal256: + } + break; + case DateTime: + if (size >= 2) { // same as DateTime64 + column.scale = Integer.parseInt(column.parameters.get(0)); + column.timeZone = TimeZone.getTimeZone(column.parameters.get(1).replace("'", "")); + } else if (size == 1) { // same as DateTime32 + // unfortunately this will fall back to GMT if the time zone + // cannot be resolved + TimeZone tz = TimeZone.getTimeZone(column.parameters.get(0).replace("'", "")); + column.timeZone = tz; + } + break; + case DateTime32: + if (size > 0) { + // unfortunately this will fall back to GMT if the time zone + // cannot be resolved + TimeZone tz = TimeZone.getTimeZone(column.parameters.get(0).replace("'", "")); + column.timeZone = tz; + } + break; + case DateTime64: + if (size > 0) { column.scale = Integer.parseInt(column.parameters.get(0)); - break; - case FixedString: + } + if (size > 1) { + column.timeZone = TimeZone.getTimeZone(column.parameters.get(1).replace("'", "")); + } + break; + case Decimal: + if (size >= 2) { column.precision = Integer.parseInt(column.parameters.get(0)); - break; - default: - break; + column.scale = Integer.parseInt(column.parameters.get(1)); + } + break; + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + column.scale = Integer.parseInt(column.parameters.get(0)); + break; + case FixedString: + column.precision = Integer.parseInt(column.parameters.get(0)); + break; + default: + break; } return column; @@ -130,8 +140,22 @@ protected static int readColumn(String args, int startIndex, int len, String nam } List params = new LinkedList<>(); i = ClickHouseUtils.readParameters(args, index, len, params); + + ClickHouseAggregateFunction aggFunc = null; + boolean isFirst = true; + List nestedColumns = new LinkedList<>(); + for (String p : params) { + if (isFirst) { + int pIndex = p.indexOf('('); + aggFunc = ClickHouseAggregateFunction.of(pIndex > 0 ? p.substring(0, pIndex) : p); + isFirst = false; + } else { + nestedColumns.add(ClickHouseColumn.of("", p)); + } + } column = new ClickHouseColumn(ClickHouseDataType.AggregateFunction, name, args.substring(startIndex, i), - nullable, lowCardinality, params, null); + nullable, lowCardinality, params, nestedColumns); + column.aggFuncType = aggFunc; } else if (args.startsWith(KEYWORD_ARRAY, i)) { int index = args.indexOf('(', i + KEYWORD_ARRAY.length()); if (index < i) { @@ -140,6 +164,10 @@ protected static int readColumn(String args, int startIndex, int len, String nam int endIndex = ClickHouseUtils.skipBrackets(args, index, len, '('); List nestedColumns = new LinkedList<>(); readColumn(args, index + 1, endIndex - 1, "", nestedColumns); + if (nestedColumns.size() != 1) { + throw new IllegalArgumentException( + "Array can have one and only one nested column, but we got: " + nestedColumns.size()); + } column = new ClickHouseColumn(ClickHouseDataType.Array, name, args.substring(startIndex, endIndex), nullable, lowCardinality, null, nestedColumns); i = endIndex; @@ -158,6 +186,10 @@ protected static int readColumn(String args, int startIndex, int len, String nam i = readColumn(args, i, endIndex, "", nestedColumns) - 1; } } + if (nestedColumns.size() != 2) { + throw new IllegalArgumentException( + "Map should have two nested columns(key and value), but we got: " + nestedColumns.size()); + } column = new ClickHouseColumn(ClickHouseDataType.Map, name, args.substring(startIndex, endIndex), nullable, lowCardinality, null, nestedColumns); i = endIndex; @@ -168,8 +200,12 @@ protected static int readColumn(String args, int startIndex, int len, String nam } i = ClickHouseUtils.skipBrackets(args, index, len, '('); String originalTypeName = args.substring(startIndex, i); + List nestedColumns = parse(args.substring(index + 1, i - 1)); + if (nestedColumns.isEmpty()) { + throw new IllegalArgumentException("Nested should have at least one nested column"); + } column = new ClickHouseColumn(ClickHouseDataType.Nested, name, originalTypeName, nullable, lowCardinality, - null, parse(args.substring(index + 1, i - 1))); + null, nestedColumns); } else if (args.startsWith(KEYWORD_TUPLE, i)) { int index = args.indexOf('(', i + KEYWORD_TUPLE.length()); if (index < i) { @@ -185,7 +221,9 @@ protected static int readColumn(String args, int startIndex, int len, String nam i = readColumn(args, i, endIndex, "", nestedColumns) - 1; } } - + if (nestedColumns.isEmpty()) { + throw new IllegalArgumentException("Tuple should have at least one nested column"); + } column = new ClickHouseColumn(ClickHouseDataType.Tuple, name, args.substring(startIndex, endIndex), nullable, lowCardinality, null, nestedColumns); } @@ -248,6 +286,14 @@ protected static int readColumn(String args, int startIndex, int len, String nam return i; } + public static ClickHouseColumn of(String columnName, ClickHouseDataType dataType, boolean nullable, int precision, + int scale) { + ClickHouseColumn column = new ClickHouseColumn(dataType, columnName, null, nullable, false, null, null); + column.precision = precision; + column.scale = scale; + return column; + } + public static ClickHouseColumn of(String columnName, ClickHouseDataType dataType, boolean nullable, boolean lowCardinality, String... parameters) { return new ClickHouseColumn(dataType, columnName, null, nullable, lowCardinality, Arrays.asList(parameters), @@ -255,9 +301,8 @@ public static ClickHouseColumn of(String columnName, ClickHouseDataType dataType } public static ClickHouseColumn of(String columnName, ClickHouseDataType dataType, boolean nullable, - boolean lowCardinality, ClickHouseColumn... nestedColumns) { - return new ClickHouseColumn(dataType, columnName, null, nullable, lowCardinality, null, - Arrays.asList(nestedColumns)); + ClickHouseColumn... nestedColumns) { + return new ClickHouseColumn(dataType, columnName, null, nullable, false, null, Arrays.asList(nestedColumns)); } public static ClickHouseColumn of(String columnName, String columnType) { @@ -317,6 +362,7 @@ private ClickHouseColumn(String originalTypeName, String columnName) { private ClickHouseColumn(ClickHouseDataType dataType, String columnName, String originalTypeName, boolean nullable, boolean lowCardinality, List parameters, List nestedColumns) { + this.aggFuncType = null; this.dataType = ClickHouseChecker.nonNull(dataType, "dataType"); this.columnName = columnName == null ? "" : columnName; @@ -324,7 +370,7 @@ private ClickHouseColumn(ClickHouseDataType dataType, String columnName, String this.nullable = nullable; this.lowCardinality = lowCardinality; - if (parameters == null || parameters.size() == 0) { + if (parameters == null || parameters.isEmpty()) { this.parameters = Collections.emptyList(); } else { List list = new ArrayList<>(parameters.size()); @@ -332,7 +378,7 @@ private ClickHouseColumn(ClickHouseDataType dataType, String columnName, String this.parameters = Collections.unmodifiableList(list); } - if (nestedColumns == null || nestedColumns.size() == 0) { + if (nestedColumns == null || nestedColumns.isEmpty()) { this.nested = Collections.emptyList(); } else { List list = new ArrayList<>(nestedColumns.size()); @@ -341,6 +387,34 @@ private ClickHouseColumn(ClickHouseDataType dataType, String columnName, String } } + public boolean isAggregateFunction() { + return dataType == ClickHouseDataType.AggregateFunction; + } + + public boolean isArray() { + return dataType == ClickHouseDataType.Array; + } + + public boolean isMap() { + return dataType == ClickHouseDataType.Map; + } + + public boolean isNested() { + return dataType == ClickHouseDataType.Nested; + } + + public boolean isTuple() { + return dataType == ClickHouseDataType.Tuple; + } + + public int getArrayNestedLevel() { + return arrayLevel; + } + + public ClickHouseColumn getArrayBaseColumn() { + return arrayBaseColumn; + } + public ClickHouseDataType getDataType() { return dataType; } @@ -361,14 +435,14 @@ boolean isLowCardinality() { return lowCardinality; } - public ClickHouseDataType getEffectiveDataType() { - return baseType != null ? baseType : dataType; - } - public TimeZone getTimeZone() { return timeZone; // null means server timezone } + public TimeZone getTimeZoneOrDefault(TimeZone defaultTz) { + return timeZone != null ? timeZone : defaultTz; + } + public int getPrecision() { return precision; } @@ -397,8 +471,68 @@ public ClickHouseColumn getValueInfo() { return dataType == ClickHouseDataType.Map && nested.size() == 2 ? nested.get(1) : null; } - public String getFunctionName() { - return dataType == ClickHouseDataType.AggregateFunction && parameters.size() > 0 ? parameters.get(0) : null; + /** + * Gets function when column type is + * {@link ClickHouseDataType#AggregateFunction}. So it will return + * {@code quantiles(0.5, 0.9)} when the column type is + * {@code AggregateFunction(quantiles(0.5, 0.9), UInt64)}. + * + * @return function, null when column type is not AggregateFunction + */ + public String getFunction() { + return dataType == ClickHouseDataType.AggregateFunction ? parameters.get(0) : null; + } + + /** + * Gets aggregate function when column type is + * {@link ClickHouseDataType#AggregateFunction}. So it will return + * {@link ClickHouseAggregateFunction#quantile} when the column type is + * {@code AggregateFunction(quantiles(0.5, 0.9), UInt64)}. + * + * @return function name, null when column type is not AggregateFunction + */ + public ClickHouseAggregateFunction getAggregateFunction() { + return aggFuncType; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((arrayBaseColumn == null) ? 0 : arrayBaseColumn.hashCode()); + result = prime * result + ((aggFuncType == null) ? 0 : aggFuncType.hashCode()); + result = prime * result + arrayLevel; + result = prime * result + ((columnName == null) ? 0 : columnName.hashCode()); + result = prime * result + ((dataType == null) ? 0 : dataType.hashCode()); + result = prime * result + (lowCardinality ? 1231 : 1237); + result = prime * result + ((nested == null) ? 0 : nested.hashCode()); + result = prime * result + (nullable ? 1231 : 1237); + result = prime * result + ((originalTypeName == null) ? 0 : originalTypeName.hashCode()); + result = prime * result + ((parameters == null) ? 0 : parameters.hashCode()); + result = prime * result + precision; + result = prime * result + scale; + result = prime * result + ((timeZone == null) ? 0 : timeZone.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + ClickHouseColumn other = (ClickHouseColumn) obj; + return Objects.equals(arrayBaseColumn, other.arrayBaseColumn) && aggFuncType == other.aggFuncType + && arrayLevel == other.arrayLevel && Objects.equals(columnName, other.columnName) + && dataType == other.dataType && lowCardinality == other.lowCardinality + && Objects.equals(nested, other.nested) && nullable == other.nullable + && Objects.equals(originalTypeName, other.originalTypeName) + && Objects.equals(parameters, other.parameters) && precision == other.precision && scale == other.scale + && Objects.equals(timeZone, other.timeZone); } @Override diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java index 8b7330598..a5ef3a895 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java @@ -9,8 +9,10 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TimeZone; + import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseConfigOption; +import com.clickhouse.client.config.ClickHouseOption; import com.clickhouse.client.config.ClickHouseDefaults; import com.clickhouse.client.config.ClickHouseSslMode; @@ -19,27 +21,28 @@ * {@link ClickHouseCredentials} and {@link ClickHouseNodeSelector} etc. */ public class ClickHouseConfig implements Serializable { - protected static final Map mergeOptions(List list) { - Map options = new HashMap<>(); - - if (list != null) { - List cl = new ArrayList<>(list.size()); - for (ClickHouseConfig c : list) { - if (c != null) { - boolean duplicated = false; - for (ClickHouseConfig conf : cl) { - if (conf == c) { - duplicated = true; - break; - } - } + protected static final Map mergeOptions(List list) { + if (list == null || list.isEmpty()) { + return Collections.emptyMap(); + } - if (duplicated) { - continue; + Map options = new HashMap<>(); + List cl = new ArrayList<>(list.size()); + for (ClickHouseConfig c : list) { + if (c != null) { + boolean duplicated = false; + for (ClickHouseConfig conf : cl) { + if (conf == c) { + duplicated = true; + break; } - options.putAll(c.options); - cl.add(c); } + + if (duplicated) { + continue; + } + options.putAll(c.options); + cl.add(c); } } @@ -47,14 +50,15 @@ protected static final Map mergeOptions(List list) { - ClickHouseCredentials credentials = null; + if (list == null || list.isEmpty()) { + return null; + } - if (list != null) { - for (ClickHouseConfig c : list) { - if (c != null && c.credentials != null) { - credentials = c.credentials; - break; - } + ClickHouseCredentials credentials = null; + for (ClickHouseConfig c : list) { + if (c != null && c.credentials != null) { + credentials = c.credentials; + break; } } @@ -62,14 +66,15 @@ protected static final ClickHouseCredentials mergeCredentials(List list) { - ClickHouseNodeSelector nodeSelector = null; + if (list == null || list.isEmpty()) { + return null; + } - if (list != null) { - for (ClickHouseConfig c : list) { - if (c != null && c.nodeSelector != null) { - nodeSelector = c.nodeSelector; - break; - } + ClickHouseNodeSelector nodeSelector = null; + for (ClickHouseConfig c : list) { + if (c != null && c.nodeSelector != null) { + nodeSelector = c.nodeSelector; + break; } } @@ -77,14 +82,15 @@ protected static final ClickHouseNodeSelector mergeNodeSelector(List list) { - Object metricRegistry = null; + if (list == null || list.isEmpty()) { + return null; + } - if (list != null) { - for (ClickHouseConfig c : list) { - if (c != null && c.metricRegistry.isPresent()) { - metricRegistry = c.metricRegistry.get(); - break; - } + Object metricRegistry = null; + for (ClickHouseConfig c : list) { + if (c != null && c.metricRegistry.isPresent()) { + metricRegistry = c.metricRegistry.get(); + break; } } @@ -96,7 +102,12 @@ protected static final Object mergeMetricRegistry(List list) { // common options optimized for read private final boolean async; private final String clientName; - private final ClickHouseCompression compression; + private final boolean compressServerResponse; + private final ClickHouseCompression compressAlgorithm; + private final int compressLevel; + private final boolean decompressClientRequest; + private final ClickHouseCompression decompressAlgorithm; + private final int decompressLevel; private final int connectionTimeout; private final String database; private final ClickHouseFormat format; @@ -108,9 +119,12 @@ protected static final Object mergeMetricRegistry(List list) { private final int maxThreads; private final boolean retry; private final boolean reuseValueWrapper; - private final int socketTimeout; + private final boolean serverInfo; + private final TimeZone serverTimeZone; + private final ClickHouseVersion serverVersion; private final int sessionTimeout; private final boolean sessionCheck; + private final int socketTimeout; private final boolean ssl; private final ClickHouseSslMode sslMode; private final String sslRootCert; @@ -122,7 +136,7 @@ protected static final Object mergeMetricRegistry(List list) { private final boolean useServerTimeZoneForDate; // client specific options - private final Map options; + private final Map options; private final ClickHouseCredentials credentials; private final transient Optional metricRegistry; @@ -156,7 +170,7 @@ public ClickHouseConfig(List configs) { * @param nodeSelector node selector * @param metricRegistry metric registry */ - public ClickHouseConfig(Map options, ClickHouseCredentials credentials, + public ClickHouseConfig(Map options, ClickHouseCredentials credentials, ClickHouseNodeSelector nodeSelector, Object metricRegistry) { this.options = new HashMap<>(); if (options != null) { @@ -165,12 +179,15 @@ public ClickHouseConfig(Map options, ClickHouseC this.async = (boolean) getOption(ClickHouseClientOption.ASYNC, ClickHouseDefaults.ASYNC); this.clientName = (String) getOption(ClickHouseClientOption.CLIENT_NAME); - this.compression = ClickHouseCompression - .fromEncoding((String) getOption(ClickHouseClientOption.COMPRESSION, ClickHouseDefaults.COMPRESSION)); + this.compressServerResponse = (boolean) getOption(ClickHouseClientOption.COMPRESS); + this.compressAlgorithm = (ClickHouseCompression) getOption(ClickHouseClientOption.COMPRESS_ALGORITHM); + this.compressLevel = (int) getOption(ClickHouseClientOption.COMPRESS_LEVEL); + this.decompressClientRequest = (boolean) getOption(ClickHouseClientOption.DECOMPRESS); + this.decompressAlgorithm = (ClickHouseCompression) getOption(ClickHouseClientOption.DECOMPRESS_ALGORITHM); + this.decompressLevel = (int) getOption(ClickHouseClientOption.DECOMPRESS_LEVEL); this.connectionTimeout = (int) getOption(ClickHouseClientOption.CONNECTION_TIMEOUT); this.database = (String) getOption(ClickHouseClientOption.DATABASE, ClickHouseDefaults.DATABASE); - this.format = ClickHouseFormat - .valueOf((String) getOption(ClickHouseClientOption.FORMAT, ClickHouseDefaults.FORMAT)); + this.format = (ClickHouseFormat) getOption(ClickHouseClientOption.FORMAT, ClickHouseDefaults.FORMAT); this.maxBufferSize = (int) getOption(ClickHouseClientOption.MAX_BUFFER_SIZE); this.maxExecutionTime = (int) getOption(ClickHouseClientOption.MAX_EXECUTION_TIME); this.maxQueuedBuffers = (int) getOption(ClickHouseClientOption.MAX_QUEUED_BUFFERS); @@ -179,11 +196,17 @@ public ClickHouseConfig(Map options, ClickHouseC this.maxThreads = (int) getOption(ClickHouseClientOption.MAX_THREADS_PER_CLIENT); this.retry = (boolean) getOption(ClickHouseClientOption.RETRY); this.reuseValueWrapper = (boolean) getOption(ClickHouseClientOption.REUSE_VALUE_WRAPPER); - this.socketTimeout = (int) getOption(ClickHouseClientOption.SOCKET_TIMEOUT); + this.serverInfo = !ClickHouseChecker.isNullOrBlank((String) getOption(ClickHouseClientOption.SERVER_TIME_ZONE)) + && !ClickHouseChecker.isNullOrBlank((String) getOption(ClickHouseClientOption.SERVER_VERSION)); + this.serverTimeZone = TimeZone.getTimeZone( + (String) getOption(ClickHouseClientOption.SERVER_TIME_ZONE, ClickHouseDefaults.SERVER_TIME_ZONE)); + this.serverVersion = ClickHouseVersion + .of((String) getOption(ClickHouseClientOption.SERVER_VERSION, ClickHouseDefaults.SERVER_VERSION)); this.sessionTimeout = (int) getOption(ClickHouseClientOption.SESSION_TIMEOUT); this.sessionCheck = (boolean) getOption(ClickHouseClientOption.SESSION_CHECK); + this.socketTimeout = (int) getOption(ClickHouseClientOption.SOCKET_TIMEOUT); this.ssl = (boolean) getOption(ClickHouseClientOption.SSL); - this.sslMode = ClickHouseSslMode.valueOf(((String) getOption(ClickHouseClientOption.SSL_MODE)).toUpperCase()); + this.sslMode = (ClickHouseSslMode) getOption(ClickHouseClientOption.SSL_MODE); this.sslRootCert = (String) getOption(ClickHouseClientOption.SSL_ROOT_CERTIFICATE); this.sslCert = (String) getOption(ClickHouseClientOption.SSL_CERTIFICATE); this.sslKey = (String) getOption(ClickHouseClientOption.SSL_KEY); @@ -210,8 +233,28 @@ public String getClientName() { return clientName; } - public ClickHouseCompression getCompression() { - return compression; + public boolean isCompressServerResponse() { + return compressServerResponse; + } + + public ClickHouseCompression getCompressAlgorithmForServerResponse() { + return compressAlgorithm; + } + + public int getCompressLevelForServerResponse() { + return compressLevel; + } + + public boolean isDecompressClientRequet() { + return decompressClientRequest; + } + + public ClickHouseCompression getDecompressAlgorithmForClientRequest() { + return decompressAlgorithm; + } + + public int getDecompressLevelForClientRequest() { + return decompressLevel; } public int getConnectionTimeout() { @@ -258,8 +301,21 @@ public boolean isReuseValueWrapper() { return reuseValueWrapper; } - public int getSocketTimeout() { - return socketTimeout; + /** + * Checks whether we got all server information(e.g. timezone and version). + * + * @return true if we got all server information; false otherwise + */ + public boolean hasServerInfo() { + return serverInfo; + } + + public TimeZone getServerTimeZone() { + return serverTimeZone; + } + + public ClickHouseVersion getServerVersion() { + return serverVersion; } public int getSessionTimeout() { @@ -270,6 +326,10 @@ public boolean isSessionCheck() { return sessionCheck; } + public int getSocketTimeout() { + return socketTimeout; + } + public boolean isSsl() { return ssl; } @@ -326,15 +386,15 @@ public Set getPreferredTags() { return this.nodeSelector.getPreferredTags(); } - public Map getAllOptions() { + public Map getAllOptions() { return Collections.unmodifiableMap(this.options); } - public Object getOption(ClickHouseConfigOption option) { + public Serializable getOption(ClickHouseOption option) { return getOption(option, null); } - public Object getOption(ClickHouseConfigOption option, ClickHouseDefaults defaultValue) { + public Serializable getOption(ClickHouseOption option, ClickHouseDefaults defaultValue) { return this.options.getOrDefault(ClickHouseChecker.nonNull(option, "option"), defaultValue == null ? option.getEffectiveDefaultValue() : defaultValue.getEffectiveDefaultValue()); } @@ -345,7 +405,7 @@ public Object getOption(ClickHouseConfigOption option, ClickHouseDefaults defaul * @param option option to test * @return true if the option is configured; false otherwise */ - public boolean hasOption(ClickHouseConfigOption option) { + public boolean hasOption(ClickHouseOption option) { return option != null && this.options.containsKey(option); } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCredentials.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCredentials.java index 51fa68aac..4adadf380 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCredentials.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCredentials.java @@ -57,8 +57,8 @@ protected ClickHouseCredentials(String accessToken) { protected ClickHouseCredentials(String userName, String password) { this.accessToken = null; - this.userName = ClickHouseChecker.nonNull(userName, "userName"); - this.password = ClickHouseChecker.nonNull(password, "password"); + this.userName = ClickHouseChecker.nonBlank(userName, "userName"); + this.password = password != null ? password : ""; } public boolean useAccessToken() { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataProcessor.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataProcessor.java index 67d029124..357f8c1d4 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataProcessor.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataProcessor.java @@ -69,7 +69,7 @@ protected ClickHouseDataProcessor(ClickHouseConfig config, InputStream input, Ou this.input = input; this.output = output; - if (settings == null || settings.size() == 0) { + if (settings == null || settings.isEmpty()) { this.settings = Collections.emptyMap(); } else { Map map = new HashMap<>(); @@ -91,7 +91,7 @@ protected ClickHouseDataProcessor(ClickHouseConfig config, InputStream input, Ou } /** - * Get column list. + * Gets list of columns to process. * * @return list of columns to process */ @@ -100,9 +100,12 @@ public final List getColumns() { } /** - * Return iterable to walk through all records in a for-each loop. + * Returns an iterable collection of records which can be walked through in a + * foreach loop. Please pay attention that: 1) + * {@link java.io.UncheckedIOException} might be thrown when iterating through + * the collection; and 2) it's not supposed to be called for more than once. * - * @return iterable + * @return non-null iterable collection */ public abstract Iterable records(); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataStreamFactory.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataStreamFactory.java index 14897a100..c9744ea3c 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataStreamFactory.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataStreamFactory.java @@ -46,7 +46,7 @@ public ClickHouseDataProcessor getProcessor(ClickHouseConfig config, InputStream ClickHouseDataProcessor processor; if (ClickHouseFormat.RowBinary == format || ClickHouseFormat.RowBinaryWithNamesAndTypes == format) { processor = new ClickHouseRowBinaryProcessor(config, input, output, columns, settings); - } else if (ClickHouseFormat.TabSeparated == format || ClickHouseFormat.TabSeparatedRaw == format + } else if (ClickHouseFormat.TSVWithNames == format || ClickHouseFormat.TSVWithNamesAndTypes == format || ClickHouseFormat.TabSeparatedWithNames == format || ClickHouseFormat.TabSeparatedWithNamesAndTypes == format) { processor = new ClickHouseTabSeparatedProcessor(config, input, output, columns, settings); @@ -67,6 +67,8 @@ public ClickHouseDataProcessor getProcessor(ClickHouseConfig config, InputStream * @return piped stream */ public ClickHousePipedStream createPipedStream(ClickHouseConfig config) { + ClickHouseChecker.nonNull(config, "config"); + return new ClickHousePipedStream(config.getMaxBufferSize(), config.getMaxQueuedBuffers(), config.getSocketTimeout()); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataType.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataType.java index 1ee9ce825..f99b771c9 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataType.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataType.java @@ -31,50 +31,57 @@ */ @SuppressWarnings("squid:S115") public enum ClickHouseDataType { - IntervalYear(Long.class, false, true, true, 8, 19, 0), IntervalQuarter(Long.class, false, true, true, 8, 19, 0), - IntervalMonth(Long.class, false, true, true, 8, 19, 0), IntervalWeek(Long.class, false, true, true, 8, 19, 0), - IntervalDay(Long.class, false, true, true, 8, 19, 0), IntervalHour(Long.class, false, true, true, 8, 19, 0), - IntervalMinute(Long.class, false, true, true, 8, 19, 0), IntervalSecond(Long.class, false, true, true, 8, 19, 0), - UInt8(Short.class, false, true, false, 1, 3, 0, "INT1 UNSIGNED", "TINYINT UNSIGNED"), - UInt16(Integer.class, false, true, false, 2, 5, 0, "SMALLINT UNSIGNED"), - UInt32(Long.class, false, true, false, 4, 10, 0, "INT UNSIGNED", "INTEGER UNSIGNED", "MEDIUMINT UNSIGNED"), - UInt64(BigInteger.class, false, true, false, 8, 19, 0, "BIGINT UNSIGNED"), - UInt128(BigInteger.class, false, true, false, 16, 20, 0), UInt256(BigInteger.class, false, true, false, 32, 39, 0), - Int8(Byte.class, false, true, true, 1, 4, 0, "BOOL", "BOOLEAN", "BYTE", "INT1", "INT1 SIGNED", "TINYINT", - "TINYINT SIGNED"), - Int16(Short.class, false, true, true, 2, 6, 0, "SMALLINT", "SMALLINT SIGNED"), - Int32(Integer.class, false, true, true, 4, 11, 0, "INT", "INTEGER", "MEDIUMINT", "INT SIGNED", "INTEGER SIGNED", - "MEDIUMINT SIGNED"), - Int64(Long.class, false, true, true, 8, 20, 0, "BIGINT", "BIGINT SIGNED"), - Int128(BigInteger.class, false, true, true, 16, 20, 0), Int256(BigInteger.class, false, true, true, 32, 40, 0), - Date(LocalDate.class, false, false, false, 2, 10, 0), Date32(LocalDate.class, false, false, false, 4, 10, 0), - DateTime(LocalDateTime.class, true, false, false, 0, 19, 0, "TIMESTAMP"), - DateTime32(LocalDateTime.class, true, false, false, 8, 19, 0), - DateTime64(LocalDateTime.class, true, false, false, 16, 38, 3), // scale up to 18 - Decimal(BigDecimal.class, true, false, true, 0, 0, 0, "DEC", "NUMERIC", "FIXED"), - Decimal32(BigDecimal.class, true, false, true, 4, 9, 9), Decimal64(BigDecimal.class, true, false, true, 8, 18, 18), - Decimal128(BigDecimal.class, true, false, true, 16, 38, 38), - Decimal256(BigDecimal.class, true, false, true, 32, 76, 20), UUID(UUID.class, false, true, false, 16, 36, 0), - Enum(String.class, true, true, false, 1, 0, 0), Enum8(String.class, true, true, false, 1, 0, 0), - Enum16(String.class, true, true, false, 2, 0, 0), - Float32(Float.class, false, true, true, 4, 8, 8, "FLOAT", "REAL", "SINGLE"), - Float64(Double.class, false, true, true, 16, 17, 17, "DOUBLE", "DOUBLE PRECISION"), - IPv4(Inet4Address.class, false, true, false, 4, 10, 0, "INET4"), - IPv6(Inet6Address.class, false, true, false, 16, 0, 0, "INET6"), - FixedString(String.class, true, true, false, 0, -1, 0, "BINARY"), - String(String.class, false, true, false, 0, 0, 0, "BINARY LARGE OBJECT", "BINARY VARYING", "BLOB", "BYTEA", "CHAR", - "CHAR LARGE OBJECT", "CHAR VARYING", "CHARACTER", "CHARACTER LARGE OBJECT", "CHARACTER VARYING", "CLOB", - "LONGBLOB", "LONGTEXT", "MEDIUMBLOB", "MEDIUMTEXT", "NATIONAL CHAR", "NATIONAL CHAR VARYING", + IntervalYear(Long.class, false, true, true, 8, 19, 0, 0, 0), + IntervalQuarter(Long.class, false, true, true, 8, 19, 0, 0, 0), + IntervalMonth(Long.class, false, true, true, 8, 19, 0, 0, 0), + IntervalWeek(Long.class, false, true, true, 8, 19, 0, 0, 0), + IntervalDay(Long.class, false, true, true, 8, 19, 0, 0, 0), + IntervalHour(Long.class, false, true, true, 8, 19, 0, 0, 0), + IntervalMinute(Long.class, false, true, true, 8, 19, 0, 0, 0), + IntervalSecond(Long.class, false, true, true, 8, 19, 0, 0, 0), + UInt8(Short.class, false, true, false, 1, 3, 0, 0, 0, "INT1 UNSIGNED", "TINYINT UNSIGNED"), + UInt16(Integer.class, false, true, false, 2, 5, 0, 0, 0, "SMALLINT UNSIGNED"), + UInt32(Long.class, false, true, false, 4, 10, 0, 0, 0, "INT UNSIGNED", "INTEGER UNSIGNED", "MEDIUMINT UNSIGNED"), + UInt64(Long.class, false, true, false, 8, 20, 0, 0, 0, "BIGINT UNSIGNED"), + UInt128(BigInteger.class, false, true, false, 16, 39, 0, 0, 0), + UInt256(BigInteger.class, false, true, false, 32, 78, 0, 0, 0), Int8(Byte.class, false, true, true, 1, 3, 0, 0, 0, + "BOOL", "BOOLEAN", "BYTE", "INT1", "INT1 SIGNED", "TINYINT", "TINYINT SIGNED"), + Int16(Short.class, false, true, true, 2, 5, 0, 0, 0, "SMALLINT", "SMALLINT SIGNED"), + Int32(Integer.class, false, true, true, 4, 10, 0, 0, 0, "INT", "INTEGER", "MEDIUMINT", "INT SIGNED", + "INTEGER SIGNED", "MEDIUMINT SIGNED"), + Int64(Long.class, false, true, true, 8, 19, 0, 0, 0, "BIGINT", "BIGINT SIGNED"), + Int128(BigInteger.class, false, true, true, 16, 39, 0, 0, 0), + Int256(BigInteger.class, false, true, true, 32, 77, 0, 0, 0), + Date(LocalDate.class, false, false, false, 2, 10, 0, 0, 0), + Date32(LocalDate.class, false, false, false, 4, 10, 0, 0, 0), + DateTime(LocalDateTime.class, true, false, false, 0, 29, 0, 0, 9, "TIMESTAMP"), + DateTime32(LocalDateTime.class, true, false, false, 4, 19, 0, 0, 0), + DateTime64(LocalDateTime.class, true, false, false, 8, 29, 3, 0, 9), + Decimal(BigDecimal.class, true, false, true, 0, 76, 0, 0, 76, "DEC", "NUMERIC", "FIXED"), + Decimal32(BigDecimal.class, true, false, true, 4, 9, 9, 0, 9), + Decimal64(BigDecimal.class, true, false, true, 8, 18, 18, 0, 18), + Decimal128(BigDecimal.class, true, false, true, 16, 38, 38, 0, 38), + Decimal256(BigDecimal.class, true, false, true, 32, 76, 20, 0, 76), + UUID(UUID.class, false, true, false, 16, 69, 0, 0, 0), Enum(String.class, true, true, false, 1, 0, 0, 0, 0), + Enum8(String.class, true, true, false, 1, 0, 0, 0, 0), Enum16(String.class, true, true, false, 2, 0, 0, 0, 0), + Float32(Float.class, false, true, true, 4, 12, 0, 0, 38, "FLOAT", "REAL", "SINGLE"), + Float64(Double.class, false, true, true, 16, 22, 0, 0, 308, "DOUBLE", "DOUBLE PRECISION"), + IPv4(Inet4Address.class, false, true, false, 4, 0, 0, 0, 0, "INET4"), + IPv6(Inet6Address.class, false, true, false, 16, 0, 0, 0, 0, "INET6"), + FixedString(String.class, true, true, false, 0, 0, 0, 0, 0, "BINARY"), + String(String.class, false, true, false, 0, 0, 0, 0, 0, "BINARY LARGE OBJECT", "BINARY VARYING", "BLOB", "BYTEA", + "CHAR", "CHAR LARGE OBJECT", "CHAR VARYING", "CHARACTER", "CHARACTER LARGE OBJECT", "CHARACTER VARYING", + "CLOB", "LONGBLOB", "LONGTEXT", "MEDIUMBLOB", "MEDIUMTEXT", "NATIONAL CHAR", "NATIONAL CHAR VARYING", "NATIONAL CHARACTER", "NATIONAL CHARACTER LARGE OBJECT", "NATIONAL CHARACTER VARYING", "NCHAR", "NCHAR LARGE OBJECT", "NCHAR VARYING", "NVARCHAR", "TEXT", "TINYBLOB", "TINYTEXT", "VARCHAR", "VARCHAR2"), - AggregateFunction(String.class, true, true, false, 0, 0, 0), // implementation-defined intermediate state - Array(Object.class, true, true, false, 0, 0, 0), Map(Map.class, true, true, false, 0, 0, 0), - Nested(Object.class, true, true, false, 0, 0, 0), Tuple(List.class, true, true, false, 0, 0, 0), - Point(Object.class, false, true, true, 33, 17, 17), // same as Tuple(Float64, Float64) - Polygon(Object.class, false, true, true, 0, 0, 0), // same as Array(Ring) - MultiPolygon(Object.class, false, true, true, 0, 0, 0), // same as Array(Polygon) - Ring(Object.class, false, true, true, 0, 0, 0), // same as Array(Point) - Nothing(Object.class, false, true, false, 0, 0, 0); + AggregateFunction(String.class, true, true, false, 0, 0, 0, 0, 0), // implementation-defined intermediate state + Array(Object.class, true, true, false, 0, 0, 0, 0, 0), Map(Map.class, true, true, false, 0, 0, 0, 0, 0), + Nested(Object.class, true, true, false, 0, 0, 0, 0, 0), Tuple(List.class, true, true, false, 0, 0, 0, 0, 0), + Point(Object.class, false, true, true, 33, 0, 0, 0, 0), // same as Tuple(Float64, Float64) + Polygon(Object.class, false, true, true, 0, 0, 0, 0, 0), // same as Array(Ring) + MultiPolygon(Object.class, false, true, true, 0, 0, 0, 0, 0), // same as Array(Polygon) + Ring(Object.class, false, true, true, 0, 0, 0, 0, 0), // same as Array(Point) + Nothing(Object.class, false, true, false, 0, 0, 0, 0, 0); /** * Immutable set(sorted) for all aliases. @@ -180,24 +187,84 @@ public static ClickHouseDataType of(String typeName) { return type; } - private final Class javaClass; + /** + * Converts given Java class to wrapper object(e.g. {@code int.class} to + * {@code Integer.class}) if applicable. + * + * @param javaClass Java class + * @return wrapper object + */ + public static Class toObjectType(Class javaClass) { + if (byte.class == javaClass || boolean.class == javaClass || Boolean.class == javaClass) { + javaClass = Byte.class; + } else if (short.class == javaClass) { + javaClass = Short.class; + } else if (int.class == javaClass || char.class == javaClass || Character.class == javaClass) { + javaClass = Integer.class; + } else if (long.class == javaClass) { + javaClass = Long.class; + } else if (float.class == javaClass) { + javaClass = Float.class; + } else if (double.class == javaClass) { + javaClass = Double.class; + } else if (javaClass == null) { + javaClass = Object.class; + } + + return javaClass; + } + + /** + * Converts given Java class to primitive types(e.g. {@code Integer.class} to + * {@code int.class}) if applicable. + * + * @param javaClass Java class + * @return primitive type + */ + public static Class toPrimitiveType(Class javaClass) { + if (Byte.class == javaClass || Boolean.class == javaClass || boolean.class == javaClass) { + javaClass = byte.class; + } else if (Short.class == javaClass) { + javaClass = short.class; + } else if (Integer.class == javaClass || Character.class == javaClass || char.class == javaClass) { + javaClass = int.class; + } else if (Long.class == javaClass) { + javaClass = long.class; + } else if (Float.class == javaClass) { + javaClass = float.class; + } else if (Double.class == javaClass) { + javaClass = double.class; + } else if (javaClass == null) { + javaClass = Object.class; + } + + return javaClass; + } + + private final Class objectType; + private final Class primitiveType; private final boolean parameter; private final boolean caseSensitive; private final boolean signed; private final List aliases; private final int byteLength; - private final int defaultPrecision; + private final int maxPrecision; private final int defaultScale; + private final int minScale; + private final int maxScale; ClickHouseDataType(Class javaClass, boolean parameter, boolean caseSensitive, boolean signed, int byteLength, - int defaultPrecision, int defaultScale, String... aliases) { - this.javaClass = javaClass == null ? Object.class : javaClass; + int maxPrecision, int defaultScale, int minScale, int maxScale, String... aliases) { + this.objectType = toObjectType(javaClass); + this.primitiveType = toPrimitiveType(javaClass); this.parameter = parameter; this.caseSensitive = caseSensitive; this.signed = signed; this.byteLength = byteLength; - this.defaultPrecision = defaultPrecision; + this.maxPrecision = maxPrecision; this.defaultScale = defaultScale; + this.minScale = minScale; + this.maxScale = maxScale; if (aliases == null || aliases.length == 0) { this.aliases = Collections.emptyList(); } else { @@ -206,12 +273,23 @@ public static ClickHouseDataType of(String typeName) { } /** - * Gets Java class for this data type. + * Gets Java class for this data type. Prefer wrapper objects to primitives(e.g. + * {@code Integer.class} instead of {@code int.class}). * * @return Java class */ - public Class getJavaClass() { - return javaClass; + public Class getObjectClass() { + return objectType; + } + + /** + * Gets Java class for this data type. Prefer primitives to wrapper objects(e.g. + * {@code int.class} instead of {@code Integer.class}). + * + * @return Java class + */ + public Class getPrimitiveClass() { + return primitiveType; } /** @@ -269,13 +347,13 @@ public int getByteLength() { } /** - * Gets default precision of this data type. Zero means unknown or not + * Gets maximum precision of this data type. Zero means unknown or not * supported. * - * @return default precision of this data type. + * @return maximum precision of this data type. */ - public int getDefaultPrecision() { - return defaultPrecision; + public int getMaxPrecision() { + return maxPrecision; } /** @@ -286,4 +364,22 @@ public int getDefaultPrecision() { public int getDefaultScale() { return defaultScale; } + + /** + * Gets minimum scale of this data type. Zero means unknown or not supported. + * + * @return minimum scale of this data type. + */ + public int getMinScale() { + return minScale; + } + + /** + * Gets maximum scale of this data type. Zero means unknown or not supported. + * + * @return maximum scale of this data type. + */ + public int getMaxScale() { + return maxScale; + } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDeserializer.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDeserializer.java index b74e774df..8378a705c 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDeserializer.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDeserializer.java @@ -13,10 +13,11 @@ public interface ClickHouseDeserializer { * * @param ref wrapper object can be reused, could be null(always return new * wrapper object) + * @param config non-null configuration * @param column non-null type information * @param input non-null input stream * @return deserialized value which might be the same instance as {@code ref} * @throws IOException when failed to read data from input stream */ - T deserialize(T ref, ClickHouseColumn column, InputStream input) throws IOException; + T deserialize(T ref, ClickHouseConfig config, ClickHouseColumn column, InputStream input) throws IOException; } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseException.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseException.java new file mode 100644 index 000000000..6904f1895 --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseException.java @@ -0,0 +1,162 @@ +package com.clickhouse.client; + +import java.net.ConnectException; +import java.net.SocketTimeoutException; +import java.util.concurrent.TimeoutException; + +/** + * Exception thrown from ClickHouse server. See full list at + * https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp. + */ +public class ClickHouseException extends Exception { + /** + * Generated ID. + */ + private static final long serialVersionUID = -2417038200885554382L; + + public static final int ERROR_ABORTED = 236; + public static final int ERROR_CANCELLED = 394; + public static final int ERROR_NETWORK = 210; + public static final int ERROR_POCO = 1000; + public static final int ERROR_TIMEOUT = 159; + public static final int ERROR_UNKNOWN = 1002; + + private final int errorCode; + + private static String buildErrorMessage(int code, Throwable cause, ClickHouseNode server) { + return buildErrorMessage(code, cause != null ? cause.getMessage() : null, server); + } + + private static String buildErrorMessage(int code, String message, ClickHouseNode server) { + StringBuilder builder = new StringBuilder(); + + if (message != null && !message.isEmpty()) { + builder.append(message); + } else { + builder.append("Unknown error ").append(code); + } + + if (server != null) { + builder.append(" on server ").append(server); + } + + return builder.toString(); + } + + private static int extractErrorCode(String errorMessage) { + if (errorMessage == null || errorMessage.isEmpty()) { + return ERROR_UNKNOWN; + } else if (errorMessage.startsWith("Poco::Exception. Code: 1000, ")) { + return ERROR_POCO; + } + + int startIndex = errorMessage.indexOf(' '); + if (startIndex >= 0) { + for (int i = ++startIndex, len = errorMessage.length(); i < len; i++) { + char ch = errorMessage.charAt(i); + if (ch == '.' || ch == ',' || Character.isWhitespace(ch)) { + try { + return Integer.parseInt(errorMessage.substring(startIndex, i)); + } catch (NumberFormatException e) { + // ignore + } + break; + } + } + } + + // this is confusing as usually it's a client-side exception + return ERROR_UNKNOWN; + } + + /** + * Creates an exception for cancellation. + * + * @param e exception + * @param server server + * @return ClickHouseException + */ + public static ClickHouseException forCancellation(Exception e, ClickHouseNode server) { + Throwable cause = e.getCause(); + if (cause == null) { + cause = e; + } + + return new ClickHouseException(ERROR_ABORTED, cause, server); + } + + /** + * Creates an exception to encapsulate cause of the given exception. + * + * @param e exception + * @param server server + * @return ClickHouseException + */ + public static ClickHouseException of(Throwable e, ClickHouseNode server) { + if (e instanceof ClickHouseException) { + return (ClickHouseException) e; + } + + Throwable cause = e != null ? e.getCause() : e; + if (cause instanceof ClickHouseException) { + return (ClickHouseException) cause; + } else if (cause == null) { + cause = e; + } + + ClickHouseException exp; + if (cause instanceof SocketTimeoutException || cause instanceof TimeoutException) { + exp = new ClickHouseException(ERROR_TIMEOUT, cause, server); + } else if (cause instanceof ConnectException) { + exp = new ClickHouseException(ERROR_NETWORK, cause, server); + } else { + exp = new ClickHouseException(extractErrorCode(cause != null ? cause.getMessage() : null), cause, server); + } + + return exp; + } + + /** + * Creates an exception to encapsulate the given error message. + * + * @param message error message + * @param server server + * @return ClickHouseException + */ + public static ClickHouseException of(String message, ClickHouseNode server) { + return new ClickHouseException(extractErrorCode(message), message, server); + } + + /** + * Constructs an exception with cause. + * + * @param code error code + * @param cause cause of the exception + * @param server server + */ + public ClickHouseException(int code, Throwable cause, ClickHouseNode server) { + super(buildErrorMessage(code, cause, server), cause); + + errorCode = code; + } + + /** + * Constructs an exception without cause. + * + * @param code error code + * @param message error message + * @param server server + */ + public ClickHouseException(int code, String message, ClickHouseNode server) { + super(buildErrorMessage(code, message, server), null); + + errorCode = code; + } + + /** + * Gets error code. + */ + public int getErrorCode() { + return errorCode; + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseFormat.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseFormat.java index fdd685465..c8c20eb2e 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseFormat.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseFormat.java @@ -2,61 +2,63 @@ /** * All formats supported by ClickHouse. More information at: - * https://clickhouse.tech/docs/en/interfaces/formats/. + * https://clickhouse.com/docs/en/interfaces/formats/. */ public enum ClickHouseFormat { - Arrow(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#arrow - ArrowStream(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#arrowstream - Avro(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#avro - AvroConfluent(true, false, true, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#avroconfluent - CapnProto(true, false, true, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#capnproto - CSV(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#csv - CSVWithNames(true, true, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#csvwithnames - CustomSeparated(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#customseparated - JSON(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#json - JSONAsString(true, false, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#jsonasstring - JSONCompact(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoncompact - JSONCompactEachRow(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoncompacteachrow - JSONCompactEachRowWithNamesAndTypes(true, true, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoncompacteachrowwithnamesandtypes - JSONCompactString(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoncompactstring - JSONCompactStringEachRow(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoncompactstringeachrow - JSONCompactStringEachRowWithNamesAndTypes(true, true, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoncompactstringeachrowwithnamesandtypes - JSONEachRow(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoneachrow - JSONEachRowWithProgress(false, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsoneachrowwithprogress - JSONString(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#jsonstring - JSONStringsEachRow(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsonstringseachrow - JSONStringsEachRowWithProgress(false, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#jsonstringseachrowwithprogress - LineAsString(true, false, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#lineasstring - Native(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#native - Null(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#null - ORC(true, false, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#orc - Parquet(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#parquet - Pretty(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#pretty - PrettyCompact(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#prettycompact - PrettyCompactMonoBlock(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#prettycompactmonoblock - PrettyNoEscapes(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#prettynoescapes - PrettySpace(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#prettyspace - Protobuf(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#protobuf - ProtobufSingle(true, true, true, true, false), // https://clickhouse.tech/docs/en/interfaces/formats/#protobufsingle - RawBLOB(true, true, true, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#rawblob - Regexp(true, false, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#regexp - RowBinary(true, true, true, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#rowbinary - RowBinaryWithNamesAndTypes(true, true, true, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#rowbinarywithnamesandtypes - TabSeparated(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#tabseparated - TabSeparatedRaw(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#tabseparatedraw - TabSeparatedWithNames(true, true, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#tabseparatedwithnames - TabSeparatedWithNamesAndTypes(true, true, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#tabseparatedwithnamesandtypes + Arrow(true, true, true, true, false), // https://clickhouse.com/docs/en/interfaces/formats/#arrow + ArrowStream(true, true, true, true, false), // https://clickhouse.com/docs/en/interfaces/formats/#arrowstream + Avro(true, true, true, true, false), // https://clickhouse.com/docs/en/interfaces/formats/#avro + AvroConfluent(true, false, true, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#avroconfluent + CSV(true, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#csv + CSVWithNames(true, true, false, true, true), // https://clickhouse.com/docs/en/interfaces/formats/#csvwithnames + CapnProto(true, false, true, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#capnproto + CustomSeparated(true, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#customseparated + CustomSeparatedIgnoreSpaces(true, true, false, false, true), JSON(false, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#json + JSONAsString(true, false, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#jsonasstring + JSONCompact(false, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#jsoncompact + JSONCompactEachRow(true, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#jsoncompacteachrow + JSONCompactEachRowWithNamesAndTypes(true, true, false, true, true), // https://clickhouse.com/docs/en/interfaces/formats/#jsoncompacteachrowwithnamesandtypes + JSONCompactStrings(false, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#jsoncompactstrings + JSONCompactStringsEachRow(true, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#jsoncompactstringeachrow + JSONCompactStringsEachRowWithNamesAndTypes(true, true, false, true, true), // https://clickhouse.com/docs/en/interfaces/formats/#jsoncompactstringeachrowwithnamesandtypes + JSONEachRow(true, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#jsoneachrow + JSONEachRowWithProgress(false, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#jsoneachrowwithprogress + JSONStringEachRow(false, false, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#jsoneachrow + JSONStrings(false, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#jsonstring + JSONStringsEachRow(true, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#jsonstringseachrow + JSONStringsEachRowWithProgress(false, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#jsonstringseachrowwithprogress + LineAsString(true, false, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#lineasstring + Markdown(false, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#lineasstring + MsgPack(true, true, true, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#msgpack + MySQLWire(false, true, true, false, false), Native(true, true, true, true, false), // https://clickhouse.com/docs/en/interfaces/formats/#native + Null(false, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#null + ODBCDriver2(false, true, true, false, false), ORC(true, false, true, true, false), // https://clickhouse.com/docs/en/interfaces/formats/#orc + Parquet(true, true, true, true, false), // https://clickhouse.com/docs/en/interfaces/formats/#parquet + PostgreSQLWire(false, true, true, false, false), Pretty(false, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#pretty + PrettyCompact(false, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#prettycompact + PrettyCompactMonoBlock(false, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#prettycompactmonoblock + PrettyCompactNoEscapes(false, true, false, false, false), PrettyNoEscapes(false, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#prettynoescapes + PrettySpace(false, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#prettyspace + PrettySpaceNoEscapes(false, true, false, false, false), Protobuf(true, true, true, true, false), // https://clickhouse.com/docs/en/interfaces/formats/#protobuf + ProtobufSingle(true, true, true, true, false), // https://clickhouse.com/docs/en/interfaces/formats/#protobufsingle + RawBLOB(true, true, true, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#rawblob + Regexp(true, false, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#regexp + RowBinary(true, true, true, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#rowbinary + RowBinaryWithNamesAndTypes(true, true, true, true, true), // https://clickhouse.com/docs/en/interfaces/formats/#rowbinarywithnamesandtypes + TSKV(true, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#tskv TSV(true, true, false, false, true), // alias of TabSeparated TSVRaw(true, true, false, false, true), // alias of TabSeparatedRaw TSVWithNames(true, true, false, true, true), // alias of TabSeparatedWithNames TSVWithNamesAndTypes(true, true, false, true, true), // alias of TabSeparatedWithNamesAndTypes - Template(true, true, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#template - TemplateIgnoreSpaces(true, false, false, true, true), // https://clickhouse.tech/docs/en/interfaces/formats/#templateignorespaces - TSKV(true, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#tskv - Values(true, true, false, false, true), // https://clickhouse.tech/docs/en/interfaces/formats/#values - Vertical(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#vertical - VerticalRaw(false, true, false, false, false), // https://clickhouse.tech/docs/en/interfaces/formats/#verticalraw - XML(false, true, false, false, false); // https://clickhouse.tech/docs/en/interfaces/formats/#xml + TabSeparated(true, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#tabseparated + TabSeparatedRaw(true, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#tabseparatedraw + TabSeparatedWithNames(true, true, false, true, true), // https://clickhouse.com/docs/en/interfaces/formats/#tabseparatedwithnames + TabSeparatedWithNamesAndTypes(true, true, false, true, true), // https://clickhouse.com/docs/en/interfaces/formats/#tabseparatedwithnamesandtypes + Template(true, true, false, true, true), // https://clickhouse.com/docs/en/interfaces/formats/#template + TemplateIgnoreSpaces(true, false, false, true, true), // https://clickhouse.com/docs/en/interfaces/formats/#templateignorespaces + Values(true, true, false, false, true), // https://clickhouse.com/docs/en/interfaces/formats/#values + Vertical(false, true, false, false, false), // https://clickhouse.com/docs/en/interfaces/formats/#vertical + XML(false, true, false, false, false); // https://clickhouse.com/docs/en/interfaces/formats/#xml private boolean input; private boolean output; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java index 99165ee71..dbbca616d 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java @@ -8,6 +8,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TimeZone; import java.util.function.BiConsumer; import java.util.function.Function; import com.clickhouse.client.config.ClickHouseDefaults; @@ -39,6 +40,9 @@ public static class Builder { protected Set tags; protected Integer weight; + protected TimeZone tz; + protected ClickHouseVersion version; + /** * Default constructor. */ @@ -72,7 +76,7 @@ protected InetSocketAddress getAddress() { protected ClickHouseProtocol getProtocol() { if (protocol == null) { - protocol = ClickHouseProtocol.valueOf((String) ClickHouseDefaults.PROTOCOL.getEffectiveDefaultValue()); + protocol = (ClickHouseProtocol) ClickHouseDefaults.PROTOCOL.getEffectiveDefaultValue(); } return protocol; } @@ -82,9 +86,6 @@ protected ClickHouseCredentials getCredentials() { } protected String getDatabase() { - if (database == null) { - database = (String) ClickHouseDefaults.DATABASE.getEffectiveDefaultValue(); - } return database; } @@ -107,6 +108,14 @@ protected int getWeight() { return weight.intValue(); } + protected TimeZone getTimeZone() { + return tz; + } + + protected ClickHouseVersion getVersion() { + return version; + } + /** * Sets cluster name. * @@ -312,6 +321,50 @@ public Builder weight(Integer weight) { return this; } + /** + * Sets time zone of this node. + * + * @param tz time zone ID, could be null + * @return this builder + */ + public Builder timeZone(String tz) { + this.tz = tz != null ? TimeZone.getTimeZone(tz) : null; + return this; + } + + /** + * Sets time zone of this node. + * + * @param tz time zone, could be null + * @return this builder + */ + public Builder timeZone(TimeZone tz) { + this.tz = tz; + return this; + } + + /** + * Sets vesion of this node. + * + * @param version version string, could be null + * @return this builder + */ + public Builder version(String version) { + this.version = version != null ? ClickHouseVersion.of(version) : null; + return this; + } + + /** + * Sets vesion of this node. + * + * @param version version, could be null + * @return this builder + */ + public Builder version(ClickHouseVersion version) { + this.version = version; + return this; + } + /** * Creates a new node. * @@ -341,8 +394,9 @@ public static Builder builder(ClickHouseNode base) { Builder b = new Builder(); if (base != null) { b.cluster(base.getCluster()).host(base.getHost()).port(base.getProtocol(), base.getPort()) - .credentials(base.getCredentials().orElse(null)).database(base.getDatabase()).tags(base.getTags()) - .weight(base.getWeight()); + .credentials(base.getCredentials().orElse(null)).database(base.getDatabase().orElse(null)) + .tags(base.getTags()).weight(base.getWeight()).timeZone(base.getTimeZone().orElse(null)) + .version(base.getVersion().orElse(null)); } return b; } @@ -375,12 +429,17 @@ public static ClickHouseNode of(String host, ClickHouseProtocol protocol, int po private final String cluster; private final ClickHouseProtocol protocol; private final InetSocketAddress address; - private final Optional credentials; + private final ClickHouseCredentials credentials; private final String database; private final Set tags; private final int weight; - private BiConsumer manager; + // extended attributes, better to use a map and offload to sub class? + private final TimeZone tz; + private final ClickHouseVersion version; + // TODO: metrics + + private transient BiConsumer manager; protected ClickHouseNode(Builder builder) { ClickHouseChecker.nonNull(builder, "builder"); @@ -388,11 +447,14 @@ protected ClickHouseNode(Builder builder) { this.cluster = builder.getCluster(); this.protocol = builder.getProtocol(); this.address = builder.getAddress(); - this.credentials = Optional.ofNullable(builder.getCredentials()); + this.credentials = builder.getCredentials(); this.database = builder.getDatabase(); this.tags = builder.getTags(); this.weight = builder.getWeight(); + this.tz = builder.getTimeZone(); + this.version = builder.getVersion(); + this.manager = null; } @@ -412,7 +474,19 @@ public InetSocketAddress getAddress() { * @return credentials for accessing this node */ public Optional getCredentials() { - return this.credentials; + return Optional.ofNullable(credentials); + } + + /** + * Gets credentials for accessing this node. It first attempts to use + * credentials tied to the node, and then use default credentials from the given + * configuration. + * + * @param config non-null configuration for retrieving default credentials + * @return credentials for accessing this node + */ + public ClickHouseCredentials getCredentials(ClickHouseConfig config) { + return credentials != null ? credentials : ClickHouseChecker.nonNull(config, "config").getDefaultCredentials(); } /** @@ -438,8 +512,19 @@ public int getPort() { * * @return database of the node */ - public String getDatabase() { - return this.database; + public Optional getDatabase() { + return Optional.ofNullable(database); + } + + /** + * Gets database of the node. When {@link #hasPreferredDatabase()} is + * {@code false}, it will use database from the given configuration. + * + * @param config non-null configuration to get default database + * @return database of the node + */ + public String getDatabase(ClickHouseConfig config) { + return hasPreferredDatabase() ? database : ClickHouseChecker.nonNull(config, "config").getDatabase(); } /** @@ -460,6 +545,46 @@ public int getWeight() { return this.weight; } + /** + * Gets time zone of the node. + * + * @return time zone of the node + */ + public Optional getTimeZone() { + return Optional.ofNullable(tz); + } + + /** + * Gets time zone of the node. When not defined, it will use server time zone + * from the given configuration. + * + * @param config non-null configuration to get server time zone + * @return time zone of the node + */ + public TimeZone getTimeZone(ClickHouseConfig config) { + return tz != null ? tz : ClickHouseChecker.nonNull(config, "config").getServerTimeZone(); + } + + /** + * Gets version of the node. + * + * @return version of the node + */ + public Optional getVersion() { + return Optional.ofNullable(version); + } + + /** + * Gets version of the node. When not defined, it will use server version from + * the given configuration. + * + * @param config non-null configuration to get server version + * @return version of the node + */ + public ClickHouseVersion getVersion(ClickHouseConfig config) { + return version != null ? version : ClickHouseChecker.nonNull(config, "config").getServerVersion(); + } + /** * Gets cluster name of the node. * @@ -478,6 +603,16 @@ public ClickHouseProtocol getProtocol() { return this.protocol; } + /** + * Checks if preferred database was specified or not. When preferred database + * was not specified, {@link #getDatabase()} will return default database. + * + * @return true if preferred database was specified; false otherwise + */ + public boolean hasPreferredDatabase() { + return database != null && !database.isEmpty(); + } + /** * Sets manager for this node. * @@ -518,15 +653,32 @@ public ClickHouseNode apply(ClickHouseNodeSelector t) { @Override public String toString() { - return new StringBuilder().append(getClass().getSimpleName()).append('(').append("cluster=").append(cluster) - .append(", protocol=").append(protocol).append(", address=").append(address).append(", database=") - .append(database).append(", tags=").append(tags).append(", weight=").append(weight).append(')') - .append('@').append(hashCode()).toString(); + StringBuilder builder = new StringBuilder().append(getClass().getSimpleName()).append('('); + boolean hasCluster = cluster != null && !cluster.isEmpty(); + if (hasCluster) { + builder.append("cluster=").append(cluster).append(", "); + } + builder.append("addr=").append(protocol.name().toLowerCase()).append(":").append(address).append(", db=") + .append(database); + if (tz != null) { + builder.append(", tz=").append(tz.getID()); + } + if (version != null) { + builder.append(", ver=").append(version); + } + if (tags != null && !tags.isEmpty()) { + builder.append(", tags=").append(tags); + } + if (hasCluster) { + builder.append(", weight=").append(weight); + } + + return builder.append(')').append('@').append(hashCode()).toString(); } @Override public int hashCode() { - return Objects.hash(address, cluster, credentials, database, protocol, tags, weight); + return Objects.hash(address, cluster, credentials, database, protocol, tags, weight, tz, version); } @Override @@ -540,8 +692,9 @@ public boolean equals(Object obj) { } ClickHouseNode node = (ClickHouseNode) obj; - return address.equals(node.address) && cluster.equals(node.cluster) && credentials.equals(node.credentials) - && database.equals(node.database) && protocol.equals(node.protocol) && tags.equals(node.tags) - && weight == node.weight; + return address.equals(node.address) && cluster.equals(node.cluster) + && Objects.equals(credentials, node.credentials) && Objects.equals(database, node.database) + && protocol == node.protocol && tags.equals(node.tags) && weight == node.weight + && Objects.equals(tz, node.tz) && Objects.equals(version, node.version); } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodeSelector.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodeSelector.java index 4975019d8..ca6b2e5c6 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodeSelector.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodeSelector.java @@ -9,6 +9,9 @@ import java.util.List; import java.util.Set; +import com.clickhouse.client.logging.Logger; +import com.clickhouse.client.logging.LoggerFactory; + /** * This class maintains two immutable lists: preferred protocols and tags. * Usually it will be used in two scenarios: 1) find suitable @@ -16,6 +19,8 @@ * suitable {@link ClickHouseNode} to connect to. */ public class ClickHouseNodeSelector implements Serializable { + private static final Logger log = LoggerFactory.getLogger(ClickHouseNodeSelector.class); + public static final ClickHouseNodeSelector EMPTY = new ClickHouseNodeSelector(Collections.emptyList(), Collections.emptyList()); @@ -56,7 +61,7 @@ public static ClickHouseNodeSelector of(String tag, String... more) { } public static ClickHouseNodeSelector of(Collection protocols, Collection tags) { - return (protocols == null || protocols.size() == 0) && (tags == null || tags.size() == 0) ? EMPTY + return (protocols == null || protocols.isEmpty()) && (tags == null || tags.isEmpty()) ? EMPTY : new ClickHouseNodeSelector(protocols, tags); } @@ -66,7 +71,7 @@ public static ClickHouseNodeSelector of(Collection protocols private final Set tags; protected ClickHouseNodeSelector(Collection protocols, Collection tags) { - if (protocols == null || protocols.size() == 0) { + if (protocols == null || protocols.isEmpty()) { this.protocols = Collections.emptyList(); } else { List p = new ArrayList<>(protocols.size()); @@ -81,10 +86,10 @@ protected ClickHouseNodeSelector(Collection protocols, Colle } } - this.protocols = p.size() == 0 ? Collections.emptyList() : Collections.unmodifiableList(p); + this.protocols = p.isEmpty() ? Collections.emptyList() : Collections.unmodifiableList(p); } - if (tags == null || tags.size() == 0) { + if (tags == null || tags.isEmpty()) { this.tags = Collections.emptySet(); } else { Set t = new HashSet<>(); @@ -96,7 +101,7 @@ protected ClickHouseNodeSelector(Collection protocols, Colle } } - this.tags = t.size() == 0 ? Collections.emptySet() : Collections.unmodifiableSet(t); + this.tags = t.isEmpty() ? Collections.emptySet() : Collections.unmodifiableSet(t); } } @@ -121,6 +126,7 @@ public boolean match(ClickHouseClient client) { if (client != null) { for (ClickHouseProtocol p : protocols) { + log.debug("Checking [%s] against [%s]...", client, p); if (client.accept(p)) { matched = true; break; @@ -142,7 +148,7 @@ public boolean match(ClickHouseNode node) { } public boolean matchAnyOfPreferredProtocols(ClickHouseProtocol protocol) { - boolean matched = protocols.size() == 0 || protocol == ClickHouseProtocol.ANY; + boolean matched = protocols.isEmpty() || protocol == ClickHouseProtocol.ANY; if (!matched && protocol != null) { for (ClickHouseProtocol p : protocols) { @@ -177,7 +183,7 @@ public boolean matchAllPreferredTags(Collection tags) { } public boolean matchAnyOfPreferredTags(Collection tags) { - boolean matched = tags.size() == 0; + boolean matched = tags.isEmpty(); if (tags != null && tags.size() > 0) { for (String t : tags) { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseParameterizedQuery.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseParameterizedQuery.java index 7ff5806a7..51278a7da 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseParameterizedQuery.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseParameterizedQuery.java @@ -6,19 +6,70 @@ import java.util.Collection; import java.util.Collections; import java.util.Iterator; -import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; /** - * A parameterized query is a parsed query with named parameters being extracted - * for substitution. + * A parameterized query is a parsed query with parameters being extracted for + * substitution. + *

+ * Here parameter is define in the format of {@code :[()]}. It + * starts with colon, followed by name, and then optionally type within + * brackets. For example: in query "select :no as no, :name(String) as name", + * both {@code no} and {@code name} are parameters. Moreover, type of the last + * parameter is {@code String}. */ -public final class ClickHouseParameterizedQuery implements Serializable { +public class ClickHouseParameterizedQuery implements Serializable { private static final long serialVersionUID = 8108887349618342152L; + /** + * A part of query. + */ + protected static class QueryPart implements Serializable { + protected final String part; + protected final int paramIndex; + protected final String paramName; + protected final ClickHouseColumn paramType; + + protected QueryPart(String part, int paramIndex, String paramName, String paramType) { + this.part = part; + this.paramIndex = paramIndex; + this.paramName = paramName != null ? paramName : String.valueOf(paramIndex); + // what should be default? ClickHouseAnyValue(simply convert object to string)? + this.paramType = paramType != null ? ClickHouseColumn.of("", paramType) : null; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + paramIndex; + result = prime * result + ((paramName == null) ? 0 : paramName.hashCode()); + result = prime * result + ((paramType == null) ? 0 : paramType.hashCode()); + result = prime * result + ((part == null) ? 0 : part.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + QueryPart other = (QueryPart) obj; + return paramIndex == other.paramIndex && Objects.equals(paramName, other.paramName) + && Objects.equals(paramType, other.paramType) && Objects.equals(part, other.part); + } + } + /** * Substitute named parameters in given SQL. * @@ -29,7 +80,7 @@ public final class ClickHouseParameterizedQuery implements Serializable { */ public static String apply(String sql, Map params) { int len = sql == null ? 0 : sql.length(); - if (len < 2 || params == null || params.size() == 0) { + if (len < 2 || params == null || params.isEmpty()) { return sql; } @@ -98,27 +149,72 @@ public static ClickHouseParameterizedQuery of(String query) { return new ClickHouseParameterizedQuery(query); } - private final String originalQuery; - // 0 - from; 1 - to; 2 - parameter index(-1 means no parameter) - private final List parts; + protected final String originalQuery; - private int[] lastPart; - private String[] names; + private final List parts; + private final Set names; + private final String lastPart; - private ClickHouseParameterizedQuery(String query) { + /** + * Default constructor. + * + * @param query non-blank query + */ + protected ClickHouseParameterizedQuery(String query) { originalQuery = ClickHouseChecker.nonBlank(query, "query"); parts = new LinkedList<>(); - lastPart = null; - names = new String[0]; + names = new LinkedHashSet<>(); + lastPart = parse(); + } - parse(); + /** + * Adds part of query and the following parameter. + * + * @param part part of the query, between previous and current parameter + * @param paramIndex zero-based index of the parameter + * @param paramType type of the parameter, could be null + */ + protected void addPart(String part, int paramIndex, String paramType) { + addPart(part, paramIndex, null, paramType); } - private void parse() { + /** + * Adds part of query and the following parameter. + * + * @param part part of the query, between previous and current parameter + * @param paramIndex zero-based index of the parameter + * @param paramName name of the parameter, null means + * {@code String.valueOf(paramIndex)} + * @param paramType type of the parameter, could be null + */ + + protected void addPart(String part, int paramIndex, String paramName, String paramType) { + if (paramName == null) { + paramName = String.valueOf(paramIndex); + } + parts.add(new QueryPart(part, paramIndex, paramName, paramType)); + names.add(paramName); + } + + /** + * Gets immutable list of query parts. + * + * @return immutable list of query parts + */ + protected List getParts() { + return Collections.unmodifiableList(parts); + } + + /** + * Parses the query given in constructor. + * + * @return remaining part(right after the last parameter) after parsing, could + * be null + */ + protected String parse() { int paramIndex = 0; int partIndex = 0; - Map params = new LinkedHashMap<>(); int len = originalQuery.length(); for (int i = 0; i < len; i++) { char ch = originalQuery.charAt(i); @@ -134,25 +230,20 @@ private void parse() { if (nextCh == ch) { // skip PostgreSQL-like type conversion i = i + 1; } else if (Character.isJavaIdentifierStart(nextCh)) { - int[] part = new int[] { partIndex, i, -1 }; - parts.add(part); + String part = partIndex != i ? originalQuery.substring(partIndex, i) : ""; + String paramName = null; + String paramType = null; StringBuilder builder = new StringBuilder().append(nextCh); for (i = i + 2; i < len; i++) { char c = originalQuery.charAt(i); - if (c == '(') { - i = ClickHouseUtils.skipBrackets(originalQuery, i, len, c); - String name = builder.toString(); - builder.setLength(0); - Integer existing = params.get(name); - if (existing == null) { - part[2] = paramIndex; - params.put(name, paramIndex++); - } else { - part[2] = existing.intValue(); - } - } else if (Character.isJavaIdentifierPart(c)) { + if (Character.isJavaIdentifierPart(c)) { builder.append(c); } else { + if (c == '(') { + int idx = ClickHouseUtils.skipBrackets(originalQuery, i, len, c); + paramType = originalQuery.substring(i + 1, idx - 1); + i = idx; + } break; } } @@ -160,29 +251,19 @@ private void parse() { partIndex = i--; if (builder.length() > 0) { - String name = builder.toString(); - Integer existing = params.get(name); - if (existing == null) { - part[2] = paramIndex; - params.put(name, paramIndex++); - } else { - part[2] = existing.intValue(); + paramName = builder.toString(); + if (names.add(paramName)) { + paramIndex++; } } + + parts.add(new QueryPart(part, paramIndex, paramName, paramType)); } } } } - names = new String[paramIndex]; - int index = 0; - for (String name : params.keySet()) { - names[index++] = name; - } - - if (partIndex < len) { - lastPart = new int[] { partIndex, len, -1 }; - } + return partIndex < len ? originalQuery.substring(partIndex, len) : null; } /** @@ -201,13 +282,13 @@ public String apply(Map params) { } StringBuilder builder = new StringBuilder(); - for (int[] part : parts) { - builder.append(originalQuery.substring(part[0], part[1])); - builder.append(params.getOrDefault(names[part[2]], ClickHouseValues.NULL_EXPR)); + for (QueryPart p : parts) { + builder.append(p.part); + builder.append(params.getOrDefault(p.paramName, ClickHouseValues.NULL_EXPR)); } if (lastPart != null) { - builder.append(originalQuery.substring(lastPart[0], lastPart[1])); + builder.append(lastPart); } return builder.toString(); } @@ -226,14 +307,14 @@ public String apply(Collection params) { StringBuilder builder = new StringBuilder(); Iterator it = params == null ? null : params.iterator(); boolean hasMore = it != null && it.hasNext(); - for (int[] part : parts) { - builder.append(originalQuery.substring(part[0], part[1])); + for (QueryPart p : parts) { + builder.append(p.part); builder.append(hasMore ? it.next() : ClickHouseValues.NULL_EXPR); hasMore = hasMore && it.hasNext(); } if (lastPart != null) { - builder.append(originalQuery.substring(lastPart[0], lastPart[1])); + builder.append(lastPart); } return builder.toString(); } @@ -255,8 +336,8 @@ public String apply(Object param, Object... more) { int len = more == null ? 0 : more.length + 1; StringBuilder builder = new StringBuilder(); int index = 0; - for (int[] part : parts) { - builder.append(originalQuery.substring(part[0], part[1])); + for (QueryPart p : parts) { + builder.append(p.part); if (index > 0) { param = index < len ? more[index - 1] : null; } @@ -265,7 +346,7 @@ public String apply(Object param, Object... more) { } if (lastPart != null) { - builder.append(originalQuery.substring(lastPart[0], lastPart[1])); + builder.append(lastPart); } return builder.toString(); } @@ -286,15 +367,15 @@ public String apply(Object[] values) { int len = values == null ? 0 : values.length; StringBuilder builder = new StringBuilder(); int index = 0; - for (int[] part : parts) { - builder.append(originalQuery.substring(part[0], part[1])); + for (QueryPart p : parts) { + builder.append(p.part); builder.append( index < len ? ClickHouseValues.convertToSqlExpression(values[index]) : ClickHouseValues.NULL_EXPR); index++; } if (lastPart != null) { - builder.append(originalQuery.substring(lastPart[0], lastPart[1])); + builder.append(lastPart); } return builder.toString(); } @@ -314,8 +395,8 @@ public String apply(String param, String... more) { int len = more == null ? 0 : more.length + 1; StringBuilder builder = new StringBuilder(); int index = 0; - for (int[] part : parts) { - builder.append(originalQuery.substring(part[0], part[1])); + for (QueryPart p : parts) { + builder.append(p.part); if (index > 0) { param = index < len ? more[index - 1] : ClickHouseValues.NULL_EXPR; } @@ -324,7 +405,7 @@ public String apply(String param, String... more) { } if (lastPart != null) { - builder.append(originalQuery.substring(lastPart[0], lastPart[1])); + builder.append(lastPart); } return builder.toString(); } @@ -343,14 +424,14 @@ public String apply(String[] values) { int len = values == null ? 0 : values.length; StringBuilder builder = new StringBuilder(); int index = 0; - for (int[] part : parts) { - builder.append(originalQuery.substring(part[0], part[1])); + for (QueryPart p : parts) { + builder.append(p.part); builder.append(index < len ? values[index] : ClickHouseValues.NULL_EXPR); index++; } if (lastPart != null) { - builder.append(originalQuery.substring(lastPart[0], lastPart[1])); + builder.append(lastPart); } return builder.toString(); } @@ -361,7 +442,7 @@ public String apply(String[] values) { * @return list of named parameters */ public List getNamedParameters() { - return names.length == 0 ? Collections.emptyList() : Arrays.asList(names); + return names.isEmpty() ? Collections.emptyList() : Arrays.asList(names.toArray(new String[0])); } /** @@ -382,34 +463,34 @@ public String getOriginalQuery() { */ public List getQueryParts() { List queryParts = new ArrayList<>(parts.size() + 1); - for (int[] part : parts) { - queryParts.add(new String[] { originalQuery.substring(part[0], part[1]), names[part[2]] }); + for (QueryPart p : parts) { + queryParts.add(new String[] { p.part, p.paramName }); } if (lastPart != null) { - queryParts.add(new String[] { originalQuery.substring(lastPart[0], lastPart[1]), null }); + queryParts.add(new String[] { lastPart, null }); } return queryParts; } /** - * Checks if the query has at least one named parameter or not. + * Checks if the query has at least one parameter or not. * - * @return true if there's at least one named parameter; false otherwise + * @return true if there's at least one parameter; false otherwise */ public boolean hasParameter() { - return names.length > 0; + return !names.isEmpty(); } @Override public int hashCode() { final int prime = 31; int result = 1; - result = prime * result + Arrays.hashCode(lastPart); - result = prime * result + Arrays.hashCode(names); - result = prime * result + ((originalQuery == null) ? 0 : originalQuery.hashCode()); - result = prime * result + ((parts == null) ? 0 : parts.hashCode()); + result = prime * result + ((lastPart == null) ? 0 : lastPart.hashCode()); + result = prime * result + names.hashCode(); + result = prime * result + originalQuery.hashCode(); + result = prime * result + parts.hashCode(); return result; } @@ -423,8 +504,7 @@ public boolean equals(Object obj) { } ClickHouseParameterizedQuery other = (ClickHouseParameterizedQuery) obj; - return Arrays.equals(lastPart, other.lastPart) && Arrays.equals(names, other.names) - && Objects.equals(originalQuery, other.originalQuery) && ((parts.isEmpty() && other.parts.isEmpty()) - || Arrays.deepEquals(parts.toArray(new int[0][]), other.parts.toArray(new int[0][]))); + return Objects.equals(lastPart, other.lastPart) && names.equals(other.names) + && originalQuery.equals(other.originalQuery) && parts.equals(other.parts); } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseProtocol.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseProtocol.java index 63b3a4295..70b69d187 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseProtocol.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseProtocol.java @@ -1,5 +1,9 @@ package com.clickhouse.client; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + /** * This defines protocols can be used to connect to ClickHouse. */ @@ -9,34 +13,69 @@ public enum ClickHouseProtocol { */ ANY(8123), /** - * HTTP/HTTPS. + * HTTP/HTTPS interface. */ - HTTP(8123), + HTTP(8123, "http", "https"), /** - * Native TCP. + * Native interface. */ - NATIVE(9000), + TCP(9000, "native", "tcp"), /** * MySQL interface. */ - MYSQL(9004), + MYSQL(9004, "mysql"), /** * PostgreSQL interface. */ - POSTGRESQL(9005), + POSTGRESQL(9005, "postgres", "postgresql", "pgsql"), /** - * Inter-server HTTP/HTTPS. + * Inter-server. */ - INTERSERVER(9009), + // INTERSERVER(9009), /** * GRPC interface. */ - GRPC(9100); + GRPC(9100, "grpc"); + + /** + * Gets most suitable protocol according to given URI scheme. + * + * @param scheme case insensitive URI scheme + * @return suitable protocol, {@link #ANY} if not found + */ + public static ClickHouseProtocol fromUriScheme(String scheme) { + ClickHouseProtocol protocol = ANY; + + for (ClickHouseProtocol p : values()) { + for (String s : p.getUriSchemes()) { + if (s.equalsIgnoreCase(scheme)) { + protocol = p; + break; + } + } + } + + return protocol; + } private final int defaultPort; + private final List schemes; - ClickHouseProtocol(int defaultPort) { + ClickHouseProtocol(int defaultPort, String... schemes) { this.defaultPort = defaultPort; + + int len = schemes != null ? schemes.length : 0; + if (len > 0) { + List list = new ArrayList<>(len); + for (String scheme : schemes) { + if (scheme != null) { + list.add(scheme); + } + } + this.schemes = Collections.unmodifiableList(list); + } else { + this.schemes = Collections.emptyList(); + } } /** @@ -47,4 +86,13 @@ public enum ClickHouseProtocol { public int getDefaultPort() { return this.defaultPort; } + + /** + * Gets supported URI schemes. + * + * @return URI schemes + */ + public List getUriSchemes() { + return this.schemes; + } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRecord.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRecord.java index 43325ee8e..9d50684d7 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRecord.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRecord.java @@ -1,8 +1,8 @@ package com.clickhouse.client; -import java.io.IOException; import java.io.Serializable; import java.util.Iterator; +import java.util.NoSuchElementException; /** * This defines a record returned from ClickHouse server. Usually it's a row but @@ -10,13 +10,6 @@ * whole data set. */ public interface ClickHouseRecord extends Iterable, Serializable { - /** - * Gets size of the record. - * - * @return size of the record - */ - int size(); - /** * Gets deserialized value wrapped in an object using column index. Please avoid * to cache the wrapper object, as it's reused among records for memory @@ -27,24 +20,22 @@ public interface ClickHouseRecord extends Iterable, Serializabl * * @param index index of the column * @return non-null wrapped value - * @throws IOException when failed get data from the input stream */ - ClickHouseValue getValue(int index) throws IOException; + ClickHouseValue getValue(int index); /** - * Gets deserialized value wrapped in an object using column name, which usually - * is slower than {@link #getValue(int)}. Please avoid to cache the wrapper - * object, as it's reused among records for memory efficiency when - * {@link ClickHouseConfig#isReuseValueWrapper()} returns {@code true}, which is - * the default value. So instead of + * Gets deserialized value wrapped in an object using case-insensitive column + * name, which usually is slower than {@link #getValue(int)}. Please avoid to + * cache the wrapper object, as it's reused among records for memory efficiency + * when {@link ClickHouseConfig#isReuseValueWrapper()} returns {@code true}, + * which is the default value. So instead of * {@code map.put("my_value", record.getValue("my_column"))}, try something like * {@code map.put("my_value", record.getValue("my_column").asString())}. * - * @param name name of the column + * @param name case-insensitive name of the column * @return non-null wrapped value - * @throws IOException when failed get data from the input stream */ - ClickHouseValue getValue(String name) throws IOException; + ClickHouseValue getValue(String name); @Override default Iterator iterator() { @@ -60,10 +51,17 @@ public boolean hasNext() { public ClickHouseValue next() { try { return getValue(index++); - } catch (IOException e) { - throw new IllegalStateException(e); + } catch (ArrayIndexOutOfBoundsException e) { + throw new NoSuchElementException(e.getMessage()); } } }; } + + /** + * Gets size of the record. + * + * @return size of the record + */ + int size(); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java index 9e2daa1d4..eb14a9fd1 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java @@ -16,13 +16,16 @@ import java.util.Objects; import java.util.Map.Entry; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; import java.util.Optional; +import java.util.Properties; import java.util.function.Function; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseConfigOption; +import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.client.config.ClickHouseDefaults; import com.clickhouse.client.data.ClickHouseExternalTable; -import com.clickhouse.client.exception.ClickHouseException; /** * Request object holding references to {@link ClickHouseClient}, @@ -97,7 +100,7 @@ public Mutation data(String file, ClickHouseCompression compression) { if (compression != null && compression != ClickHouseCompression.NONE) { // TODO create input stream } else { - this.input = fileInput; + this.input = CompletableFuture.completedFuture(fileInput); } return this; @@ -112,7 +115,7 @@ public Mutation data(String file, ClickHouseCompression compression) { public Mutation data(InputStream input) { checkSealed(); - this.input = input; + this.input = CompletableFuture.completedFuture(input); return this; } @@ -122,9 +125,9 @@ public Mutation data(InputStream input) { * {@code client.execute(request.seal())}. * * @return future to get response - * @throws ClickHouseException when error occurred + * @throws CompletionException when error occurred */ - public CompletableFuture send() throws ClickHouseException { + public CompletableFuture send() { return getClient().execute(isSealed() ? this : seal()); } @@ -178,12 +181,12 @@ public Mutation seal() { protected final ClickHouseConfig clientConfig; protected final Function server; protected final transient List externalTables; - protected final Map options; + protected final Map options; protected final Map settings; protected final Map namedParameters; - protected transient InputStream input; + protected transient CompletableFuture input; protected String queryId; protected String sessionId; protected String sql; @@ -220,7 +223,7 @@ protected void checkSealed() { protected ClickHouseClient getClient() { if (client == null) { - client = ClickHouseClient.builder().nodeSelector(clientConfig.getNodeSelector()).build(); + client = ClickHouseClient.builder().config(clientConfig).build(); } return client; @@ -301,16 +304,16 @@ public final ClickHouseNode getServer() { } /** - * Gets merged configuration. + * Gets request configuration. * - * @return merged configuration + * @return request configuration */ public ClickHouseConfig getConfig() { if (config == null) { - if (options.size() == 0) { + if (options.isEmpty()) { config = clientConfig; } else { - Map merged = new HashMap<>(); + Map merged = new HashMap<>(); merged.putAll(clientConfig.getAllOptions()); merged.putAll(options); config = new ClickHouseConfig(merged, clientConfig.getDefaultCredentials(), @@ -327,17 +330,14 @@ public ClickHouseConfig getConfig() { * @return input stream */ public Optional getInputStream() { - return Optional.ofNullable(input); - } - - /** - * Gets compression used for communication between server and client. Same as - * {@code getConfig().getCompression()}. - * - * @return compression used for communication between server and client - */ - public ClickHouseCompression getCompression() { - return getConfig().getCompression(); + try { + return Optional.ofNullable(input != null ? input.get() : null); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new CompletionException(e); + } catch (ExecutionException e) { + throw new CompletionException(e.getCause()); + } } /** @@ -430,22 +430,152 @@ public List getStatements(boolean withSettings) { } /** - * Sets preferred compression algorithm to be used between server and client. + * Enable or disable compression of server response. Pay attention that + * {@link ClickHouseClientOption#COMPRESS_ALGORITHM} and + * {@link ClickHouseClientOption#COMPRESS_LEVEL} will be used. + * + * @param enable true to enable compression of server response; false otherwise + * @return the request itself + */ + public SelfT compressServerResponse(boolean enable) { + return compressServerResponse(enable, null, + (int) ClickHouseClientOption.COMPRESS_LEVEL.getEffectiveDefaultValue()); + } + + /** + * Enable or disable compression of server response. Pay attention that + * {@link ClickHouseClientOption#COMPRESS_LEVEL} will be used. + * + * @param enable true to enable compression of server response; false + * otherwise + * @param compressAlgorithm compression algorithm, null is treated as + * {@link ClickHouseCompression#NONE} or + * {@link ClickHouseClientOption#COMPRESS_ALGORITHM} + * depending on whether enabled + * @return the request itself + */ + public SelfT compressServerResponse(boolean enable, ClickHouseCompression compressAlgorithm) { + return compressServerResponse(enable, compressAlgorithm, + (int) ClickHouseClientOption.COMPRESS_LEVEL.getEffectiveDefaultValue()); + } + + /** + * Enable or disable compression of server response. + * + * @param enable true to enable compression of server response; false + * otherwise + * @param compressAlgorithm compression algorithm, null is treated as + * {@link ClickHouseCompression#NONE} or + * {@link ClickHouseClientOption#COMPRESS_ALGORITHM} + * depending on whether enabled + * @param compressLevel compression level + * @return the request itself + */ + @SuppressWarnings("unchecked") + public SelfT compressServerResponse(boolean enable, ClickHouseCompression compressAlgorithm, int compressLevel) { + checkSealed(); + + if (compressAlgorithm == null) { + compressAlgorithm = enable + ? (ClickHouseCompression) ClickHouseClientOption.COMPRESS_ALGORITHM.getEffectiveDefaultValue() + : ClickHouseCompression.NONE; + } + + if (compressLevel < 0) { + compressLevel = 0; + } else if (compressLevel > 9) { + compressLevel = 9; + } + + Object oldSwitch = options.put(ClickHouseClientOption.COMPRESS, enable); + Object oldAlgorithm = options.put(ClickHouseClientOption.COMPRESS_ALGORITHM, compressAlgorithm); + Object oldLevel = options.put(ClickHouseClientOption.COMPRESS_LEVEL, compressLevel); + if (oldSwitch == null || !oldSwitch.equals(enable) || oldAlgorithm == null + || !oldAlgorithm.equals(compressAlgorithm) || oldLevel == null || !oldLevel.equals(compressLevel)) { + resetCache(); + } + + return (SelfT) this; + } + + /** + * Enable or disable compression of client request. Pay attention that + * {@link ClickHouseClientOption#DECOMPRESS_ALGORITHM} and + * {@link ClickHouseClientOption#DECOMPRESS_LEVEL} will be used. * - * @param compression compression used in transportation, null or - * {@link ClickHouseCompression#NONE} means no compression + * @param enable true to enable compression of client request; false otherwise + * @return the request itself + */ + public SelfT decompressClientRequest(boolean enable) { + return decompressClientRequest(enable, null, + (int) ClickHouseClientOption.DECOMPRESS_LEVEL.getEffectiveDefaultValue()); + } + + /** + * Enable or disable compression of client request. Pay attention that + * {@link ClickHouseClientOption#DECOMPRESS_LEVEL} will be used. + * + * @param enable true to enable compression of client request; false + * otherwise + * @param compressAlgorithm compression algorithm, null is treated as + * {@link ClickHouseCompression#NONE} or + * {@link ClickHouseClientOption#DECOMPRESS_ALGORITHM} + * depending on whether enabled + * @return the request itself + */ + public SelfT decompressClientRequest(boolean enable, ClickHouseCompression compressAlgorithm) { + return decompressClientRequest(enable, compressAlgorithm, + (int) ClickHouseClientOption.DECOMPRESS_LEVEL.getEffectiveDefaultValue()); + } + + /** + * Enable or disable compression of client request. + * + * @param enable true to enable compression of client request; false + * otherwise + * @param compressAlgorithm compression algorithm, null is treated as + * {@link ClickHouseCompression#NONE} + * @param compressLevel compression level * @return the request itself */ @SuppressWarnings("unchecked") - public SelfT compression(ClickHouseCompression compression) { + public SelfT decompressClientRequest(boolean enable, ClickHouseCompression compressAlgorithm, int compressLevel) { checkSealed(); - if (compression == null) { - compression = ClickHouseCompression.NONE; + if (compressAlgorithm == null) { + compressAlgorithm = enable + ? (ClickHouseCompression) ClickHouseClientOption.DECOMPRESS_ALGORITHM.getEffectiveDefaultValue() + : ClickHouseCompression.NONE; } - Object oldValue = options.put(ClickHouseClientOption.COMPRESSION, compression.name()); - if (oldValue == null || !oldValue.equals(compression.name())) { + if (compressLevel < 0) { + compressLevel = 0; + } else if (compressLevel > 9) { + compressLevel = 9; + } + + Object oldSwitch = options.put(ClickHouseClientOption.DECOMPRESS, enable); + Object oldAlgorithm = options.put(ClickHouseClientOption.DECOMPRESS_ALGORITHM, compressAlgorithm); + Object oldLevel = options.put(ClickHouseClientOption.DECOMPRESS_LEVEL, compressLevel); + if (oldSwitch == null || !oldSwitch.equals(enable) || oldAlgorithm == null + || !oldAlgorithm.equals(compressAlgorithm) || oldLevel == null || !oldLevel.equals(compressLevel)) { + resetCache(); + } + + return (SelfT) this; + } + + /** + * Adds an external table. + * + * @param table non-null external table + * @return the request itself + */ + @SuppressWarnings("unchecked") + public SelfT addExternal(ClickHouseExternalTable table) { + checkSealed(); + + if (externalTables.add(ClickHouseChecker.nonNull(table, TYPE_EXTERNAL_TABLE))) { resetCache(); } @@ -505,8 +635,8 @@ public SelfT format(ClickHouseFormat format) { checkSealed(); Object oldValue = options.put(ClickHouseClientOption.FORMAT, - ClickHouseChecker.nonNull(format, "format").name()); - if (oldValue == null || !oldValue.equals(format.name())) { + format != null ? format : (ClickHouseFormat) ClickHouseDefaults.FORMAT.getEffectiveDefaultValue()); + if (oldValue == null || !oldValue.equals(format)) { resetCache(); } @@ -522,7 +652,7 @@ public SelfT format(ClickHouseFormat format) { * @return the request itself */ @SuppressWarnings("unchecked") - public SelfT option(ClickHouseConfigOption option, Serializable value) { + public SelfT option(ClickHouseOption option, Serializable value) { checkSealed(); Object oldValue = options.put(ClickHouseChecker.nonNull(option, "option"), @@ -534,6 +664,59 @@ public SelfT option(ClickHouseConfigOption option, Serializable value) { return (SelfT) this; } + /** + * Sets all options. {@code option} is for configuring client's behaviour, while + * {@code setting} is for server. + * + * @param options options + * @return the request itself + */ + @SuppressWarnings("unchecked") + public SelfT options(Map options) { + checkSealed(); + + this.options.clear(); + if (options != null) { + this.options.putAll(options); + } + + resetCache(); + + return (SelfT) this; + } + + /** + * Sets all options. {@code option} is for configuring client's behaviour, while + * {@code setting} is for server. + * + * @param options options + * @return the request itself + */ + @SuppressWarnings("unchecked") + public SelfT options(Properties options) { + checkSealed(); + + this.options.clear(); + if (options != null) { + for (Entry e : options.entrySet()) { + Object key = e.getKey(); + Object value = e.getValue(); + if (key == null || value == null) { + continue; + } + + ClickHouseClientOption o = ClickHouseClientOption.fromKey(key.toString()); + if (o != null) { + this.options.put(o, ClickHouseOption.fromString(value.toString(), o.getValueType())); + } + } + } + + resetCache(); + + return (SelfT) this; + } + /** * Sets stringified parameters. Be aware of SQL injection risk as mentioned in * {@link #params(String, String...)}. @@ -1037,7 +1220,7 @@ public SelfT use(String database) { public SelfT removeExternal(ClickHouseExternalTable external) { checkSealed(); - if (externalTables.remove(ClickHouseChecker.nonNull(external, "external"))) { + if (externalTables.remove(ClickHouseChecker.nonNull(external, TYPE_EXTERNAL_TABLE))) { resetCache(); } @@ -1080,7 +1263,7 @@ public SelfT removeExternal(String name) { * @return the request itself */ @SuppressWarnings("unchecked") - public SelfT removeOption(ClickHouseConfigOption option) { + public SelfT removeOption(ClickHouseOption option) { checkSealed(); if (options.remove(ClickHouseChecker.nonNull(option, "option")) != null) { @@ -1175,9 +1358,9 @@ public Mutation write() { * Executes the request. Same as {@code client.execute(request.seal())}. * * @return future to get response - * @throws ClickHouseException when error occurred preparing for the execution + * @throws CompletionException when error occurred during execution */ - public CompletableFuture execute() throws ClickHouseException { - return client.execute(isSealed() ? this : seal()); + public CompletableFuture execute() { + return getClient().execute(isSealed() ? this : seal()); } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponse.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponse.java index 49ae83aaa..c9467c9bc 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponse.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponse.java @@ -4,244 +4,117 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.Serializable; -import java.util.Collections; -import java.util.Iterator; +import java.io.UncheckedIOException; import java.util.List; -import java.util.Map; +import java.util.NoSuchElementException; import java.util.Spliterator; import java.util.Spliterators; import java.util.stream.Stream; import java.util.stream.StreamSupport; -import com.clickhouse.client.exception.ClickHouseException; -import com.clickhouse.client.exception.ClickHouseExceptionSpecifier; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; /** - * This represents server reponse. To get data returned from server, depending - * on actual needs, you have 3 options: + * This encapsulates a server reponse. Depending on concrete implementation, it + * could be either an in-memory list or a wrapped input stream with + * {@link ClickHouseDataProcessor} attached for deserialization. To get data + * returned from server, depending on actual needs, you have 3 options: * *

    - *
  • use {@link #records()} or {@link #recordStream()} to get deserialized - * records(usually rows), a record is composed of one or more values
    • - *
  • use {@link #values()} or {@link #valueStream()} to get deserialized - * values
    • - *
  • use {@link #getInputStream()} for custom processing like dumping results - * into a file
    • + *
  • use {@link #records()} or {@link #stream()} to get deserialized + * {@link ClickHouseRecord} one at a time
    • + *
  • use {@link #firstRecord()} if you're certain that all you need is the + * first {@link ClickHouseRecord}
    • + *
  • use {@link #getInputStream()} or {@link #pipe(OutputStream, int)} if you + * prefer to handle stream instead of deserialized data
    • *
*/ -public class ClickHouseResponse implements AutoCloseable, Serializable { - private static final Logger log = LoggerFactory.getLogger(ClickHouseResponse.class); - - private static final long serialVersionUID = 2271296998310082447L; - - private static final ClickHouseResponseSummary emptySummary = new ClickHouseResponseSummary() { - }; - - protected static final List defaultTypes = Collections - .singletonList(ClickHouseColumn.of("results", "Nullable(String)")); - - protected final ClickHouseConfig config; - protected final ClickHouseNode server; - protected final InputStream input; - protected final ClickHouseDataProcessor processor; - protected final List columns; - protected final Throwable error; - - protected ClickHouseResponse(ClickHouseConfig config, ClickHouseNode server, Throwable error) - throws ClickHouseException { - this(config, server, null, null, null, error); - } - - protected ClickHouseResponse(ClickHouseConfig config, ClickHouseNode server, Map settings, - InputStream input, List columns, Throwable error) throws ClickHouseException { - try { - this.config = ClickHouseChecker.nonNull(config, "config"); - this.server = ClickHouseChecker.nonNull(server, "server"); - - if (error != null) { - this.processor = null; - this.input = null; - // response object may be constructed in a separate thread - this.error = error; - } else if (input == null) { - throw new IllegalArgumentException("input cannot be null when there's no error"); - } else { - this.input = input; - this.processor = ClickHouseDataStreamFactory.getInstance().getProcessor(config, input, null, settings, - columns); - - this.error = null; - } - - this.columns = columns != null ? columns - : (processor != null ? processor.getColumns() : Collections.emptyList()); - } catch (IOException | RuntimeException e) { // TODO and Error? - if (input != null) { - log.warn("Failed to instantiate response object, will try to close the given input stream"); - try { - input.close(); - } catch (IOException exp) { - log.warn("Failed to close given input stream", exp); - } - } - - throw ClickHouseExceptionSpecifier.specify(e, server); - } - } - - protected void throwErrorIfAny() throws ClickHouseException { - if (error == null) { - return; - } - - if (error instanceof ClickHouseException) { - throw (ClickHouseException) error; - } else { - throw ClickHouseExceptionSpecifier.specify(error, server); - } - } - - @Override - public void close() { - if (input != null) { - long skipped = 0L; - try { - skipped = input.skip(Long.MAX_VALUE); - log.debug("%d bytes skipped before closing input stream", skipped); - } catch (Exception e) { - // ignore - log.debug("%d bytes skipped before closing input stream", skipped, e); - } finally { - try { - input.close(); - } catch (Exception e) { - log.warn("Failed to close input stream", e); - } - } - } - } - - public boolean hasError() { - return error != null; - } - - public List getColumns() throws ClickHouseException { - throwErrorIfAny(); - - return columns; - } - - public ClickHouseFormat getFormat() throws ClickHouseException { - throwErrorIfAny(); - - return this.config.getFormat(); - } - - public ClickHouseNode getServer() { - return server; - } +public interface ClickHouseResponse extends AutoCloseable, Serializable { + /** + * Gets list of columns. + * + * @return non-null list of column + */ + List getColumns(); - public ClickHouseResponseSummary getSummary() { - return emptySummary; - } + /** + * Gets summary of this response. Keep in mind that the summary may change over + * time until response is closed. + * + * @return non-null summary of this response + */ + ClickHouseResponseSummary getSummary(); /** - * This is the most memory-efficient way for you to handle data returned from - * ClickHouse. However, this also means additional work is required for - * deserialization, especially when using a binary format. + * Gets input stream of the response. In general, this is the most + * memory-efficient way for streaming data from server to client. However, this + * also means additional work is required for deserialization, especially when + * using a binary format. * - * @return input stream get raw data returned from server - * @throws ClickHouseException when failed to get input stream or read data + * @return input stream for getting raw data returned from server */ - public InputStream getInputStream() throws ClickHouseException { - throwErrorIfAny(); + InputStream getInputStream(); - return input; + /** + * Gets the first record only. Please use {@link #records()} instead if you need + * to access the rest of records. + * + * @return the first record + * @throws NoSuchElementException when there's no record at all + * @throws UncheckedIOException when failed to read data(e.g. deserialization) + */ + default ClickHouseRecord firstRecord() { + return records().iterator().next(); } /** - * Dump response into output stream. + * Returns an iterable collection of records which can be walked through in a + * foreach loop. Please pay attention that: 1) {@link UncheckedIOException} + * might be thrown when iterating through the collection; and 2) it's not + * supposed to be called for more than once. * - * @param output output stream, which will remain open - * @throws ClickHouseException when error occurred dumping response and/or - * writing data into output stream + * @return non-null iterable collection */ - public void dump(OutputStream output) throws ClickHouseException { - throwErrorIfAny(); + Iterable records(); + /** + * Pipes the contents of this response into the given output stream. + * + * @param output non-null output stream, which will remain open + * @param bufferSize buffer size, 0 or negative value will be treated as 8192 + * @throws IOException when error occurred reading or writing data + */ + default void pipe(OutputStream output, int bufferSize) throws IOException { ClickHouseChecker.nonNull(output, "output"); - // TODO configurable buffer size - int size = 8192; - byte[] buffer = new byte[size]; - int counter = 0; - try { - while ((counter = input.read(buffer, 0, size)) >= 0) { - output.write(buffer, 0, counter); - } - } catch (IOException e) { - throw ClickHouseExceptionSpecifier.specify(e, server); + if (bufferSize <= 0) { + bufferSize = 8192; } - } - public Iterable records() throws ClickHouseException { - throwErrorIfAny(); - - if (processor == null) { - throw new UnsupportedOperationException( - "No data processor available for deserialization, please use getInputStream instead"); + byte[] buffer = new byte[bufferSize]; + int counter = 0; + while ((counter = getInputStream().read(buffer, 0, bufferSize)) >= 0) { + output.write(buffer, 0, counter); } - return processor.records(); + // caller's responsibility to call output.flush() as needed } - public Stream recordStream() throws ClickHouseException { + /** + * Gets stream of records to process. + * + * @return stream of records + */ + default Stream stream() { return StreamSupport.stream(Spliterators.spliteratorUnknownSize(records().iterator(), Spliterator.IMMUTABLE | Spliterator.NONNULL | Spliterator.ORDERED), false); } - public Iterable values() throws ClickHouseException { - throwErrorIfAny(); - - if (processor == null) { - throw new UnsupportedOperationException( - "No data processor available for deserialization, please use getInputStream instead"); - } - - return new Iterable() { - @Override - public Iterator iterator() { - return new Iterator() { - Iterator records = processor.records().iterator(); - ClickHouseRecord current = null; - int index = 0; - - @Override - public boolean hasNext() { - return records.hasNext() || (current != null && index < current.size()); - } - - @Override - public ClickHouseValue next() { - if (current == null || index == current.size()) { - current = records.next(); - index = 0; - } - - try { - return current.getValue(index++); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - }; - } - }; - } + @Override + void close(); - public Stream valueStream() throws ClickHouseException { - return StreamSupport.stream(Spliterators.spliteratorUnknownSize(values().iterator(), - Spliterator.IMMUTABLE | Spliterator.NONNULL | Spliterator.ORDERED), false); - } + /** + * Checks whether the reponse has been closed or not. + * + * @return true if the response has been closed; false otherwise + */ + boolean isClosed(); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponseSummary.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponseSummary.java index 8581ca9ac..c13581c2a 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponseSummary.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponseSummary.java @@ -1,44 +1,231 @@ package com.clickhouse.client; import java.io.Serializable; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; /** * Summary of ClickHouse response. */ -public interface ClickHouseResponseSummary extends Serializable { - default long getAllocatedBytes() { - return 0L; +public class ClickHouseResponseSummary implements Serializable { + private static final long serialVersionUID = 6241261266635143197L; + + public static final ClickHouseResponseSummary EMPTY = new ClickHouseResponseSummary(null, null, true); + + /** + * Progress of a query. + */ + public static final class Progress implements Serializable { + private static final long serialVersionUID = -1447066780591278108L; + + private final long read_rows; + private final long read_bytes; + private final long total_rows_to_read; + private final long written_rows; + private final long written_bytes; + + /** + * Default constructor. + * + * @param read_rows Number of rows read + * @param read_bytes Volume of data read in bytes + * @param total_rows_to_read Total number of rows to be read + * @param written_rows Number of rows written + * @param written_bytes Volume of data written in bytes + */ + public Progress(long read_rows, long read_bytes, long total_rows_to_read, long written_rows, + long written_bytes) { + this.read_rows = read_rows; + this.read_bytes = read_bytes; + this.total_rows_to_read = total_rows_to_read; + this.written_rows = written_rows; + this.written_bytes = written_bytes; + } + + public long getReadRows() { + return read_rows; + } + + public long getReadBytes() { + return read_bytes; + } + + public long getTotalRowsToRead() { + return total_rows_to_read; + } + + public long getWrittenRows() { + return written_rows; + } + + public long getWrittenBytes() { + return written_bytes; + } + } + + /** + * Statistics of a query. + */ + public static class Statistics implements Serializable { + private static final long serialVersionUID = -7744796632866829161L; + + private final long rows; + private final long blocks; + private final long allocated_bytes; + private final boolean applied_limit; + private final long rows_before_limit; + + /** + * Default constructor. + * + * @param rows The total number of output rows + * @param blocks + * @param allocated_bytes + * @param applied_limit + * @param rows_before_limit The minimal number of rows there would have been + * without LIMIT + */ + public Statistics(long rows, long blocks, long allocated_bytes, boolean applied_limit, long rows_before_limit) { + this.rows = rows; + this.blocks = blocks; + this.allocated_bytes = allocated_bytes; + this.applied_limit = applied_limit; + this.rows_before_limit = rows_before_limit; + } + + public long getRows() { + return rows; + } + + public long getBlocks() { + return blocks; + } + + public long getAllocatedBytes() { + return allocated_bytes; + } + + public boolean hasAppliedLimit() { + return applied_limit; + } + + public long getRowsBeforeLimit() { + return rows_before_limit; + } + } + + private final AtomicReference progress; + private final AtomicReference stats; + private final AtomicInteger updates; + + private volatile boolean sealed; + + /** + * Default constructor. + * + * @param progress progress which may or may not be null + * @param stats statistics which may or may not be null + */ + + public ClickHouseResponseSummary(Progress progress, Statistics stats) { + this(progress, stats, false); + } + + /** + * Default constructor. + * + * @param progress progress which may or may not be null + * @param stats statistics which may or may not be null + * @param sealed whether the summary is sealed + */ + protected ClickHouseResponseSummary(Progress progress, Statistics stats, boolean sealed) { + this.progress = new AtomicReference<>(progress != null ? progress : new Progress(0L, 0L, 0L, 0L, 0L)); + this.stats = new AtomicReference<>(stats != null ? stats : new Statistics(0L, 0L, 0L, false, 0L)); + this.updates = new AtomicInteger(1); + + this.sealed = sealed; + } + + /** + * Seals the object so that it cannot be updated any more. + */ + public void seal() { + this.sealed = true; + } + + /** + * Increases update counter. + * + * @return increased update counter + */ + public int update() { + return this.updates.incrementAndGet(); + } + + /** + * Updates query progress. + * + * @param progress query progress, null value will be simply ignored + */ + public void update(Progress progress) { + if (sealed) { + throw new IllegalStateException("Sealed summary cannot be updated"); + } + + if (progress != null) { + this.progress.set(progress); + } + } + + public void update(Statistics stats) { + if (sealed) { + throw new IllegalStateException("Sealed summary cannot be updated"); + } + + if (stats != null) { + this.stats.set(stats); + } } - default long getBlocks() { - return 0L; + /** + * Gets current progress of the query. + * + * @return non-null progress + */ + public Progress getProgress() { + return progress.get(); } - default long getRows() { - return 0L; + /** + * Gets statistics of the query. + * + * @return non-null statistics + */ + public Statistics getStatistics() { + return stats.get(); } - default long getRowsBeforeLimit() { - return 0L; + public long getReadRows() { + return progress.get().getReadRows(); } - default long getReadBytes() { - return 0L; + public long getReadBytes() { + return progress.get().getReadBytes(); } - default long getReadRows() { - return 0L; + public long getTotalRowsToRead() { + return progress.get().getTotalRowsToRead(); } - default long getTotalRowsToRead() { - return 0L; + public long getWrittenRows() { + return progress.get().getWrittenRows(); } - default long getWriteBytes() { - return 0L; + public long getWrittenBytes() { + return progress.get().getWrittenBytes(); } - default long getWriteRows() { - return 0L; + public int getUpdateCount() { + return updates.get(); } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSerializer.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSerializer.java index 75d1c3f8b..36d44614c 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSerializer.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSerializer.java @@ -12,9 +12,10 @@ public interface ClickHouseSerializer { * Writes serialized value to output stream. * * @param value non-null value to be serialized + * @param config non-null configuration * @param column non-null type information * @param output non-null output stream * @throws IOException when failed to write data to output stream */ - void serialize(T value, ClickHouseColumn column, OutputStream output) throws IOException; + void serialize(T value, ClickHouseConfig config, ClickHouseColumn column, OutputStream output) throws IOException; } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java index 5fe3029ec..da06b8a4e 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java @@ -7,12 +7,20 @@ import java.io.InputStream; import java.io.OutputStream; import java.lang.reflect.Array; +import java.math.BigDecimal; +import java.math.BigInteger; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Deque; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; @@ -83,7 +91,7 @@ public static String applyVariables(String template, UnaryOperator apply } public static String applyVariables(String template, Map variables) { - return applyVariables(template, variables == null || variables.size() == 0 ? null : variables::get); + return applyVariables(template, variables == null || variables.isEmpty() ? null : variables::get); } private static T findFirstService(Class serviceInterface) { @@ -206,6 +214,268 @@ public static String format(String template, Object... args) { return String.format(Locale.ROOT, template, args); } + private static int readJsonArray(String json, List array, int startIndex, int len) { + StringBuilder builder = new StringBuilder(); + + // skip the first bracket + for (int i = startIndex + 1; i < len; i++) { + char ch = json.charAt(i); + if (Character.isWhitespace(ch) || ch == ',' || ch == ':') { + continue; + } else if (ch == '"') { + i = readUnescapedJsonString(json, builder, i, len) - 1; + array.add(builder.toString()); + builder.setLength(0); + } else if (ch == '-' || (ch >= '0' && ch <= '9')) { + List list = new ArrayList<>(1); + i = readJsonNumber(json, list, i, len) - 1; + array.add(list.get(0)); + builder.setLength(0); + } else if (ch == '{') { + Map map = new LinkedHashMap<>(); + i = readJsonObject(json, map, i, len) - 1; + array.add(map); + } else if (ch == '[') { + List list = new LinkedList<>(); + i = readJsonArray(json, list, i, len) - 1; + array.add(list.toArray(new Object[0])); + } else if (ch == ']') { + return i + 1; + } else { + List list = new ArrayList<>(1); + i = readJsonConstants(json, list, i, len) - 1; + array.add(list.get(0)); + } + } + + return len; + } + + private static int readJsonConstants(String json, List value, int startIndex, int len) { + String c = "null"; + if (json.indexOf(c, startIndex) == startIndex) { + value.add(null); + } else if (json.indexOf(c = "false", startIndex) == startIndex) { + value.add(Boolean.FALSE); + } else if (json.indexOf(c = "true", startIndex) == startIndex) { + value.add(Boolean.TRUE); + } else { + throw new IllegalArgumentException(format("Expect one of 'null', 'false', 'true' but we got '%s'", + json.substring(startIndex, Math.min(startIndex + 5, len)))); + } + + return startIndex + c.length(); + } + + private static int readJsonNumber(String json, List value, int startIndex, int len) { + int endIndex = len; + + StringBuilder builder = new StringBuilder().append(json.charAt(startIndex)); + + boolean hasDot = false; + boolean hasExp = false; + // add first digit + for (int i = startIndex + 1; i < len; i++) { + char n = json.charAt(i); + if (n >= '0' && n <= '9') { + builder.append(n); + } else if (!hasDot && n == '.') { + hasDot = true; + builder.append(n); + } else if (!hasExp && (n == 'e' || n == 'E')) { + hasDot = true; + hasExp = true; + builder.append(n); + if (i + 1 < len) { + char next = json.charAt(i + 1); + if (next == '+' || next == '-') { + builder.append(next); + i++; + } + } + + boolean hasNum = false; + for (int j = i + 1; j < len; j++) { + char next = json.charAt(j); + if (next >= '0' && next <= '9') { + hasNum = true; + builder.append(next); + } else { + if (!hasNum) { + throw new IllegalArgumentException("Expect number after exponent at " + i); + } + endIndex = j + 1; + break; + } + } + break; + } else { + endIndex = i; + break; + } + } + + if (hasDot) { + if (hasExp || builder.length() >= 21) { + value.add(new BigDecimal(builder.toString())); + } else if (builder.length() >= 11) { + value.add(Double.parseDouble(builder.toString())); + } else { + value.add(Float.parseFloat(builder.toString())); + } + } else { + if (hasExp || builder.length() >= 19) { + value.add(new BigInteger(builder.toString())); + } else if (builder.length() >= 10) { + value.add(Long.parseLong(builder.toString())); + } else { + value.add(Integer.parseInt(builder.toString())); + } + } + + return endIndex; + } + + private static int readJsonObject(String json, Map object, int startIndex, int len) { + StringBuilder builder = new StringBuilder(); + + String key = null; + // skip the first bracket + for (int i = startIndex + 1; i < len; i++) { + char ch = json.charAt(i); + if (Character.isWhitespace(ch) || ch == ',' || ch == ':') { + continue; + } else if (ch == '"') { + i = readUnescapedJsonString(json, builder, i, len) - 1; + if (key != null) { + object.put(key, builder.toString()); + key = null; + } else { + key = builder.toString(); + } + builder.setLength(0); + } else if (ch == '-' || (ch >= '0' && ch <= '9')) { + if (key == null) { + throw new IllegalArgumentException("Key is not available"); + } + List list = new ArrayList<>(1); + i = readJsonNumber(json, list, i, len) - 1; + object.put(key, list.get(0)); + key = null; + builder.setLength(0); + } else if (ch == '{') { + if (key == null) { + throw new IllegalArgumentException("Key is not available"); + } + Map map = new LinkedHashMap<>(); + i = readJsonObject(json, map, i, len) - 1; + object.put(key, map); + key = null; + builder.setLength(0); + } else if (ch == '[') { + if (key == null) { + throw new IllegalArgumentException("Key is not available"); + } + + List list = new LinkedList<>(); + i = readJsonArray(json, list, i, len) - 1; + key = null; + object.put(key, list.toArray(new Object[0])); + } else if (ch == '}') { + return i + 1; + } else { + if (key == null) { + throw new IllegalArgumentException("Key is not available"); + } + List list = new ArrayList<>(1); + i = readJsonConstants(json, list, i, len) - 1; + object.put(key, list.get(0)); + key = null; + } + } + + return len; + } + + private static int readUnescapedJsonString(String json, StringBuilder builder, int startIndex, int len) { + // skip the first double quote + for (int i = startIndex + 1; i < len; i++) { + char c = json.charAt(i); + if (c == '\\') { + if (++i < len) { + builder.append(json.charAt(i)); + } + } else if (c == '"') { + return i + 1; + } else { + builder.append(c); + } + } + + return len; + } + + /** + * Simple and un-protected JSON parser. + * + * @param json non-empty JSON string + * @return object array, Boolean, Number, null, String, or Map + * @throws IllegalArgumentException when JSON string is null or empty + */ + public static Object parseJson(String json) { + if (json == null || json.isEmpty()) { + throw new IllegalArgumentException("Non-empty JSON string is required"); + } + + boolean hasValue = false; + Object value = null; + StringBuilder builder = new StringBuilder(); + for (int i = 0, len = json.length(); i < len; i++) { + char ch = json.charAt(i); + if (Character.isWhitespace(ch)) { + continue; + } else if (ch == '{') { + // read object + Map map = new LinkedHashMap<>(); + i = readJsonObject(json, map, i, len) - 1; + hasValue = true; + value = map; + } else if (ch == '[') { + // read array + List list = new LinkedList<>(); + i = readJsonArray(json, list, i, len) - 1; + hasValue = true; + value = list.toArray(new Object[0]); + } else if (ch == '"') { + // read string + i = readUnescapedJsonString(json, builder, i, len) - 1; + hasValue = true; + value = builder.toString(); + } else if (ch == '-' || (ch >= '0' && ch <= '9')) { + // read number + List list = new ArrayList<>(1); + i = readJsonNumber(json, list, i, len) - 1; + hasValue = true; + value = list.get(0); + } else { + List list = new ArrayList<>(1); + i = readJsonConstants(json, list, i, len) - 1; + hasValue = true; + value = list.get(0); + } + + if (hasValue) { + break; + } + } + + if (!hasValue) { + throw new IllegalArgumentException("No value extracted from given JSON string"); + } + + return value; + } + public static char getCloseBracket(char openBracket) { char closeBracket; if (openBracket == '(') { @@ -342,6 +612,91 @@ public static OutputStream getFileOutputStream(String file) throws IOException { return new FileOutputStream(file, false); } + /** + * Extracts key value pairs from the given string. + * + * @param str string + * @return non-null map containing extracted key value pairs + */ + public static Map getKeyValuePairs(String str) { + if (str == null || str.isEmpty()) { + return Collections.emptyMap(); + } + + Map map = new LinkedHashMap<>(); + String key = null; + StringBuilder builder = new StringBuilder(); + for (int i = 0, len = str.length(); i < len; i++) { + char ch = str.charAt(i); + if (ch == '\\' && i + 1 < len) { + ch = str.charAt(++i); + builder.append(ch); + continue; + } + + if (Character.isWhitespace(ch)) { + if (builder.length() > 0) { + builder.append(ch); + } + } else if (ch == '=' && key == null) { + key = builder.toString().trim(); + builder.setLength(0); + } else if (ch == ',' && key != null) { + String value = builder.toString().trim(); + builder.setLength(0); + if (!key.isEmpty() && !value.isEmpty()) { + map.put(key, value); + } + key = null; + } else { + builder.append(ch); + } + } + + if (key != null && builder.length() > 0) { + String value = builder.toString().trim(); + if (!key.isEmpty() && !value.isEmpty()) { + map.put(key, value); + } + } + + return Collections.unmodifiableMap(map); + } + + public static String getLeadingComment(String sql) { + if (sql == null || sql.isEmpty()) { + return ""; + } + + StringBuilder builder = new StringBuilder(); + for (int i = 0, len = sql.length(); i + 1 < len; i++) { + char ch = sql.charAt(i); + char nextCh = sql.charAt(i + 1); + + if (ch == '-' && nextCh == '-') { + int index = skipSingleLineComment(sql, i, len); + if (index > i + 2) { + builder.append(sql.substring(i + 2, index).trim()); + } + i = index - 1; + } else if (ch == '/' && nextCh == '*') { + int index = skipMultiLineComment(sql, i + 2, len); + if (index > i + 4) { + builder.append(sql.substring(i + 2, index - 2).trim()); + } + i = index - 1; + } else if (!Character.isWhitespace(ch)) { + break; + } + + if (builder.length() > 0) { + break; + } + } + + return builder.toString(); + } + public static String getProperty(String key, Properties... props) { return getProperty(key, null, props); } @@ -457,6 +812,9 @@ public static int skipSingleLineComment(String args, int startIndex, int len) { */ public static int skipMultiLineComment(String args, int startIndex, int len) { int openIndex = args.indexOf("/*", startIndex); + if (openIndex == startIndex) { + openIndex = args.indexOf("/*", startIndex + 2); + } int closeIndex = args.indexOf("*/", startIndex); if (closeIndex < startIndex) { @@ -468,22 +826,32 @@ public static int skipMultiLineComment(String args, int startIndex, int len) { } /** - * Skip quoted string, comments, and brackets until seeing {@code endChar} or - * reaching end of the given string. + * Skip quoted string, comments, and brackets until seeing one of + * {@code endChars} or reaching end of the given string. * * @param args non-null string to scan * @param startIndex start index * @param len end index, usually length of the given string - * @param endChar skip characters until seeing this or reaching end of the - * string + * @param endChars skip characters until seeing one of the specified + * characters or reaching end of the string; '\0' is used when + * it's null or empty * @return index of {@code endChar} or {@code len} */ - public static int skipContentsUntil(String args, int startIndex, int len, char endChar) { + public static int skipContentsUntil(String args, int startIndex, int len, char... endChars) { + int charLen = endChars != null ? endChars.length : 0; + if (charLen == 0) { + endChars = new char[] { '\0' }; + charLen = 1; + } for (int i = startIndex; i < len; i++) { char ch = args.charAt(i); - if (ch == endChar) { - return i + 1; - } else if (isQuote(ch)) { + for (int j = 0; j < charLen; j++) { + if (ch == endChars[j]) { + return i + 1; + } + } + + if (isQuote(ch)) { i = skipQuotedString(args, i, len, ch) - 1; } else if (isOpenBracket(ch)) { i = skipBrackets(args, i, len, ch) - 1; @@ -500,6 +868,113 @@ public static int skipContentsUntil(String args, int startIndex, int len, char e return len; } + /** + * Skip quoted string, comments, and brackets until seeing the {@code keyword} + * or reaching end of the given string. + * + * @param args non-null string to scan + * @param startIndex start index + * @param len end index, usually length of the given string + * @param keyword keyword, null or empty string means any character + * @param caseSensitive whether keyword is case sensitive or not + * @return index of {@code endChar} or {@code len} + */ + public static int skipContentsUntil(String args, int startIndex, int len, String keyword, boolean caseSensitive) { + if (keyword == null || keyword.isEmpty()) { + return Math.min(startIndex + 1, len); + } + + int k = keyword.length(); + if (k == 1) { + return skipContentsUntil(args, startIndex, len, keyword.charAt(0)); + } + + for (int i = startIndex; i < len; i++) { + char ch = args.charAt(i); + + if (isQuote(ch)) { + i = skipQuotedString(args, i, len, ch) - 1; + } else if (isOpenBracket(ch)) { + i = skipBrackets(args, i, len, ch) - 1; + } else if (i + 1 < len) { + char nextCh = args.charAt(i + 1); + if (ch == '-' && nextCh == '-') { + i = skipSingleLineComment(args, i + 2, len) - 1; + } else if (ch == '/' && nextCh == '*') { + i = skipMultiLineComment(args, i + 2, len) - 1; + } else if (i + k < len) { + int endIndex = i + k; + String s = args.substring(i, endIndex); + if ((caseSensitive && s.equals(keyword)) || (!caseSensitive && s.equalsIgnoreCase(keyword))) { + return endIndex; + } + } + } + } + + return len; + } + + /** + * Skip quoted string, comments, and brackets until seeing all the given + * {@code keywords}(with only whitespaces or comments in between) or reaching + * end of the given string. + * + * @param args non-null string to scan + * @param startIndex start index + * @param len end index, usually length of the given string + * @param keywords keywords, null or empty one means any character + * @param caseSensitive whether keyword is case sensitive or not + * @return index of {@code endChar} or {@code len} + */ + public static int skipContentsUntil(String args, int startIndex, int len, String[] keywords, + boolean caseSensitive) { + int k = keywords != null ? keywords.length : 0; + if (k == 0) { + return Math.min(startIndex + 1, len); + } + + int index = skipContentsUntil(args, startIndex, len, keywords[0], caseSensitive); + for (int j = 1; j < k; j++) { + String keyword = keywords[j]; + if (keyword == null || keyword.isEmpty()) { + index++; + continue; + } else { + int klen = keyword.length(); + if (index + klen >= len) { + return len; + } + + for (int i = index; i < len; i++) { + String s = args.substring(i, i + klen); + if ((caseSensitive && s.equals(keyword)) || (!caseSensitive && s.equalsIgnoreCase(keyword))) { + index = i + klen; + break; + } else { + char ch = args.charAt(i); + if (Character.isWhitespace(ch)) { + continue; + } else if (i + 1 < len) { + char nextCh = args.charAt(i + 1); + if (ch == '-' && nextCh == '-') { + i = skipSingleLineComment(args, i + 2, len) - 1; + } else if (ch == '/' && nextCh == '*') { + i = skipMultiLineComment(args, i + 2, len) - 1; + } else { + return len; + } + } else { + return len; + } + } + } + } + } + + return index; + } + public static int readNameOrQuotedString(String args, int startIndex, int len, StringBuilder builder) { char quote = '\0'; for (int i = startIndex; i < len; i++) { @@ -523,8 +998,9 @@ public static int readNameOrQuotedString(String args, int startIndex, int len, S } else { builder.append(ch); } - } else if (quote == '\0' - && (Character.isWhitespace(ch) || isOpenBracket(ch) || isCloseBracket(ch) || isSeparator(ch))) { + } else if (quote == '\0' && (Character.isWhitespace(ch) || isOpenBracket(ch) || isCloseBracket(ch) + || isSeparator(ch) || (i + 1 < len && ((ch == '-' && args.charAt(i + 1) == '-') + || (ch == '/' && args.charAt(i + 1) == '*'))))) { if (builder.length() > 0) { len = i; break; @@ -586,9 +1062,33 @@ public static int readParameters(String args, int startIndex, int len, List stack = new ArrayDeque<>(); StringBuilder builder = new StringBuilder(); + for (int i = startIndex; i < len; i++) { char ch = args.charAt(i); - if ((i == startIndex && ch == '(') || Character.isWhitespace(ch)) { + if (ch == '(') { + startIndex = i + 1; + break; + } else if (Character.isWhitespace(ch)) { + continue; + } else if (i + 1 < len) { + char nextCh = args.charAt(i + 1); + if (ch == '-' && nextCh == '-') { + i = skipSingleLineComment(args, i + 2, len) - 1; + } else if (ch == '/' && nextCh == '*') { + i = skipMultiLineComment(args, i + 2, len) - 1; + } else { + startIndex = i; + break; + } + } else { + startIndex = i; + break; + } + } + + for (int i = startIndex; i < len; i++) { + char ch = args.charAt(i); + if (Character.isWhitespace(ch)) { continue; } else if (isQuote(ch)) { builder.append(ch); @@ -624,6 +1124,15 @@ public static int readParameters(String args, int startIndex, int len, List T[] asArray(Class clazz) { if (isNullOrEmpty()) { - return (T[]) ClickHouseValues.EMPTY_ARRAY; + return (T[]) ClickHouseValues.EMPTY_OBJECT_ARRAY; } T[] array = (T[]) Array.newInstance(ClickHouseChecker.nonNull(clazz, ClickHouseValues.TYPE_CLASS), 1); @@ -122,6 +132,28 @@ default T[] asArray(Class clazz) { return array; } + /** + * Gets value as byte stream. It's caller's responsibility to close the stream + * at the end of reading. + * + * @return non-null byte stream for reading + */ + default InputStream asByteStream() { + byte[] bytes = isNullOrEmpty() ? new byte[0] : new byte[] { asByte() }; + return new ByteArrayInputStream(bytes); + } + + /** + * Gets value as character stream. It's caller's responsibility to close the + * tream at the end of reading. + * + * @return non-null character stream for reading + */ + default Reader asCharacterStream() { + String s = isNullOrEmpty() ? "" : asString(); + return new StringReader(s); + } + /** * Gets value as boolean. * @@ -215,7 +247,11 @@ default BigDecimal asBigDecimal() { * @return date, could be null */ default LocalDate asDate() { - return isNullOrEmpty() ? null : LocalDate.ofEpochDay(asLong()); + if (isNullOrEmpty()) { + return null; + } + + return LocalDate.ofEpochDay(asLong()); } /** @@ -224,7 +260,21 @@ default LocalDate asDate() { * @return time, could be null */ default LocalTime asTime() { - return isNullOrEmpty() ? null : LocalTime.ofSecondOfDay(asLong()); + return asTime(0); + } + + /** + * Gets value as {@link java.time.LocalTime}. + * + * @param scale scale of the date time, between 0 (second) and 9 (nano second) + * @return time, could be null + */ + default LocalTime asTime(int scale) { + if (isNullOrEmpty()) { + return null; + } + + return asDateTime(scale).toLocalTime(); } /** @@ -237,6 +287,26 @@ default LocalDateTime asDateTime() { return asDateTime(0); } + /** + * Gets value as {@link java.time.OffsetDateTime}, using default scale(usually + * 0). + * + * @return date time, could be null + */ + default OffsetDateTime asOffsetDateTime() { + return asOffsetDateTime(0); + } + + /** + * Gets value as {@link java.time.ZonedDateTime}, using default scale(usually + * 0). + * + * @return date time, could be null + */ + default ZonedDateTime asZonedDateTime() { + return asZonedDateTime(0); + } + /** * Gets value as {@link java.time.LocalDateTime}. * @@ -244,9 +314,40 @@ default LocalDateTime asDateTime() { * @return date time, could be null */ default LocalDateTime asDateTime(int scale) { - return isNullOrEmpty() ? null - : ClickHouseValues.convertToDateTime( - asBigDecimal(ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 9))); + if (isNullOrEmpty()) { + return null; + } + + return ClickHouseValues + .convertToDateTime(asBigDecimal(ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 9))); + } + + /** + * Gets value as {@link java.time.OffsetDateTime}. + * + * @param scale scale of the date time, between 0 (second) and 9 (nano second) + * @return date time, could be null + */ + default OffsetDateTime asOffsetDateTime(int scale) { + if (isNullOrEmpty()) { + return null; + } + + return asDateTime(scale).atOffset(ZoneOffset.UTC); + } + + /** + * Gets value as {@link java.time.ZonedDateTime}. + * + * @param scale scale of the date time, between 0 (second) and 9 (nano second) + * @return date time, could be null + */ + default ZonedDateTime asZonedDateTime(int scale) { + if (isNullOrEmpty()) { + return null; + } + + return asDateTime(scale).atZone(ClickHouseValues.UTC_ZONE); } /** @@ -447,7 +548,7 @@ default UUID asUuid() { ClickHouseValue resetToNullOrEmpty(); /** - * Convert the value to escaped SQL expression. For example, number 123 will be + * Converts the value to escaped SQL expression. For example, number 123 will be * converted to {@code 123}, while string "12'3" will be converted to @{code * '12\'3'}. * @@ -455,6 +556,48 @@ default UUID asUuid() { */ String toSqlExpression(); + /** + * Updates value. + * + * @param value value to update + * @return this object + */ + default ClickHouseValue update(InputStream value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + update(new InputStreamReader(value, StandardCharsets.UTF_8)); + } + + return this; + } + + /** + * Updates value. + * + * @param value value to update + * @return this object + */ + default ClickHouseValue update(Reader value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + StringBuilder builder = new StringBuilder(); + try (BufferedReader reader = new BufferedReader(value)) { + int bufferSize = 1024; + char[] buffer = new char[bufferSize]; + for (int num; (num = reader.read(buffer, 0, buffer.length)) > 0;) { + builder.append(buffer, 0, num); + } + } catch (IOException e) { + throw new IllegalArgumentException("Failed to read", e); + } + update(builder.toString()); + } + + return this; + } + /** * Updates value. * @@ -735,6 +878,16 @@ default ClickHouseValue update(LocalDateTime value) { : update(seconds); } + /** + * Updates value. + * + * @param value value to update + * @return this object + */ + default ClickHouseValue update(OffsetDateTime value) { + return update(value != null ? value.toLocalDateTime() : null); + } + /** * Updates value. * @@ -850,7 +1003,8 @@ default ClickHouseValue update(Object[] value) { * default, it's same as {@code update(String.valueOf(value))}. * *

- * Due to limitationPlease avoid to call {@link #update(Object)} here or + * Please avoid to call {@link #update(Object)} here as it will create endless + * loop. * * @param value value to update * @return this object @@ -923,6 +1077,8 @@ default ClickHouseValue update(Object value) { return update((LocalTime) value); } else if (value instanceof LocalDateTime) { return update((LocalDateTime) value); + } else if (value instanceof OffsetDateTime) { + return update((OffsetDateTime) value); } else if (value instanceof Collection) { return update((Collection) value); } else if (value instanceof Enumeration) { @@ -935,6 +1091,10 @@ default ClickHouseValue update(Object value) { return update((UUID) value); } else if (value instanceof String) { return update((String) value); + } else if (value instanceof InputStream) { + return update((InputStream) value); + } else if (value instanceof Reader) { + return update((Reader) value); } else if (value instanceof ClickHouseValue) { return update((ClickHouseValue) value); } else { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseValues.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseValues.java index 919475a8d..5a6472600 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseValues.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseValues.java @@ -1,31 +1,41 @@ package com.clickhouse.client; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Array; import java.math.BigDecimal; import java.math.BigInteger; import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; import java.net.UnknownHostException; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.temporal.ChronoField; import java.util.Collection; import java.util.Enumeration; import java.util.Map; +import java.util.TimeZone; import java.util.UUID; import java.util.Map.Entry; import com.clickhouse.client.data.ClickHouseArrayValue; import com.clickhouse.client.data.ClickHouseBigDecimalValue; import com.clickhouse.client.data.ClickHouseBigIntegerValue; +import com.clickhouse.client.data.ClickHouseBitmapValue; import com.clickhouse.client.data.ClickHouseByteValue; import com.clickhouse.client.data.ClickHouseDateTimeValue; import com.clickhouse.client.data.ClickHouseDateValue; import com.clickhouse.client.data.ClickHouseDoubleValue; +import com.clickhouse.client.data.ClickHouseEmptyValue; import com.clickhouse.client.data.ClickHouseFloatValue; import com.clickhouse.client.data.ClickHouseGeoMultiPolygonValue; import com.clickhouse.client.data.ClickHouseGeoPointValue; @@ -37,10 +47,17 @@ import com.clickhouse.client.data.ClickHouseLongValue; import com.clickhouse.client.data.ClickHouseMapValue; import com.clickhouse.client.data.ClickHouseNestedValue; +import com.clickhouse.client.data.ClickHouseOffsetDateTimeValue; import com.clickhouse.client.data.ClickHouseShortValue; import com.clickhouse.client.data.ClickHouseStringValue; import com.clickhouse.client.data.ClickHouseTupleValue; import com.clickhouse.client.data.ClickHouseUuidValue; +import com.clickhouse.client.data.array.ClickHouseByteArrayValue; +import com.clickhouse.client.data.array.ClickHouseDoubleArrayValue; +import com.clickhouse.client.data.array.ClickHouseFloatArrayValue; +import com.clickhouse.client.data.array.ClickHouseIntArrayValue; +import com.clickhouse.client.data.array.ClickHouseLongArrayValue; +import com.clickhouse.client.data.array.ClickHouseShortArrayValue; /** * Help class for dealing with values. @@ -53,7 +70,13 @@ public final class ClickHouseValues { public static final LocalDateTime DATETIME_ZERO = LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC); public static final LocalTime TIME_ZERO = LocalTime.ofSecondOfDay(0L); - public static final Object[] EMPTY_ARRAY = new Object[0]; + public static final Object[] EMPTY_OBJECT_ARRAY = new Object[0]; + public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + public static final short[] EMPTY_SHORT_ARRAY = new short[0]; + public static final int[] EMPTY_INT_ARRAY = new int[0]; + public static final long[] EMPTY_LONG_ARRAY = new long[0]; + public static final float[] EMPTY_FLOAT_ARRAY = new float[0]; + public static final double[] EMPTY_DOUBLE_ARRAY = new double[0]; public static final String EMPTY_ARRAY_EXPR = "[]"; public static final BigDecimal NANOS = new BigDecimal(BigInteger.TEN.pow(9)); @@ -64,6 +87,9 @@ public final class ClickHouseValues { public static final DateTimeFormatter DATETIME_FORMATTER = new DateTimeFormatterBuilder() .appendPattern("yyyy-MM-dd HH:mm:ss").appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true).toFormatter(); + public static final TimeZone UTC_TIMEZONE = TimeZone.getTimeZone("UTC"); + public static final ZoneId UTC_ZONE = UTC_TIMEZONE.toZoneId(); + public static final String NULL_EXPR = "NULL"; public static final String NAN_EXPR = "NaN"; public static final String INF_EXPR = "Inf"; @@ -175,6 +201,77 @@ public static LocalDateTime convertToDateTime(BigDecimal value) { return LocalDateTime.ofEpochSecond(value.longValue() - (nanoSeconds > 0 ? 1 : 0), nanoSeconds, ZoneOffset.UTC); } + /** + * Converts big decimal to date time. + * + * @param value big decimal + * @param tz time zone, null is treated as UTC + * @return date time + */ + public static ZonedDateTime convertToDateTime(BigDecimal value, TimeZone tz) { + if (value == null) { + return null; + } + + return convertToDateTime(value, tz != null ? tz.toZoneId() : UTC_ZONE); + } + + /** + * Converts big decimal to date time. + * + * @param value big decimal + * @param zone zone id, null is treated as UTC + * @return date time + */ + public static ZonedDateTime convertToDateTime(BigDecimal value, ZoneId zone) { + if (value == null) { + return null; + } + + if (zone == null) { + zone = UTC_ZONE; + } + if (value.scale() == 0) { + return Instant.ofEpochSecond(value.longValue(), 0L).atZone(zone); + } else if (value.signum() >= 0) { + return Instant.ofEpochSecond(value.longValue(), value.remainder(BigDecimal.ONE).multiply(NANOS).intValue()) + .atZone(zone); + } + + long v = NANOS.add(value.remainder(BigDecimal.ONE).multiply(NANOS)).longValue(); + int nanoSeconds = v < 1000000000L ? (int) v : 0; + + return Instant.ofEpochSecond(value.longValue() - (nanoSeconds > 0 ? 1 : 0), nanoSeconds).atZone(zone); + } + + /** + * Converts big decimal to date time. + * + * @param value big decimal + * @param offset zone offset, null is treated as {@code ZoneOffset.UTC} + * @return date time + */ + public static OffsetDateTime convertToDateTime(BigDecimal value, ZoneOffset offset) { + if (value == null) { + return null; + } + + if (offset == null) { + offset = ZoneOffset.UTC; + } + if (value.scale() == 0) { + return Instant.ofEpochSecond(value.longValue(), 0L).atOffset(offset); + } else if (value.signum() >= 0) { + return Instant.ofEpochSecond(value.longValue(), value.remainder(BigDecimal.ONE).multiply(NANOS).intValue()) + .atOffset(offset); + } + + long v = NANOS.add(value.remainder(BigDecimal.ONE).multiply(NANOS)).longValue(); + int nanoSeconds = v < 1000000000L ? (int) v : 0; + + return Instant.ofEpochSecond(value.longValue() - (nanoSeconds > 0 ? 1 : 0), nanoSeconds).atOffset(offset); + } + /** * Converts IPv6 address to IPv4 address if applicable. * @@ -367,6 +464,12 @@ public static String convertToSqlExpression(Object value) { } else if (value instanceof LocalDateTime) { s = new StringBuilder().append('\'').append(((LocalDateTime) value).format(DATETIME_FORMATTER)).append('\'') .toString(); + } else if (value instanceof OffsetDateTime) { + s = new StringBuilder().append('\'').append(((OffsetDateTime) value).format(DATETIME_FORMATTER)) + .append('\'').toString(); + } else if (value instanceof ZonedDateTime) { + s = new StringBuilder().append('\'').append(((ZonedDateTime) value).format(DATETIME_FORMATTER)).append('\'') + .toString(); } else if (value instanceof InetAddress) { s = new StringBuilder().append('\'').append(((InetAddress) value).getHostAddress()).append('\'').toString(); } else if (value instanceof Enum) { @@ -673,6 +776,63 @@ public static String convertToQuotedString(Object value) { .toString(); } + /** + * Creates an object array. Primitive types will be converted to corresponding + * wrapper types, also {@code Boolean} / {@code boolean} will be converted to + * {@code Byte}, and {@code Character} / {@code char} to {@code Integer}. + * + * @param type of the base element + * @param clazz class of the base element, null is treated as + * {@code Object.class} + * @param length length of the array, negative is treated as zero + * @param level level of the array, must between 1 and 255 + * @return a non-null object array + */ + @SuppressWarnings("unchecked") + public static T[] createObjectArray(Class clazz, int length, int level) { + if (level < 1) { + level = 1; + } else if (level > 255) { + level = 255; + } + int[] dimensions = new int[level]; + dimensions[0] = length < 0 ? 0 : length; + return (T[]) Array.newInstance(ClickHouseDataType.toObjectType(clazz), dimensions); + } + + /** + * Creates a primitive array if applicable. Wrapper types will be converted to + * corresponding primitive types, also {@code Boolean} / {@code boolean} will be + * converted to {@code byte}, and {@code Character} / {@code char} to + * {@code int}. + * + * @param clazz class of the base element + * @param length length of the array, negative is treated as zero + * @param level level of the array, must between 1 and 255 + * @return a primitive array if applicable; an object array otherwise + */ + public static Object createPrimitiveArray(Class clazz, int length, int level) { + if (level < 1) { + level = 1; + } else if (level > 255) { + level = 255; + } + int[] dimensions = new int[level]; + dimensions[0] = length < 0 ? 0 : length; + return Array.newInstance(ClickHouseDataType.toPrimitiveType(clazz), dimensions); + } + + static ClickHouseArrayValue fillByteArray(ClickHouseArrayValue array, ClickHouseConfig config, + ClickHouseColumn column, InputStream input, ClickHouseDeserializer deserializer, + int length) throws IOException { + byte[] values = new byte[length]; + ClickHouseValue ref = ClickHouseByteValue.ofNull(); + for (int i = 0; i < length; i++) { + values[i] = deserializer.deserialize(ref, config, column, input).asByte(); + } + return array.update(values); + } + /** * Extract one and only value from singleton collection. * @@ -743,115 +903,159 @@ public static ClickHouseValue newValue(ClickHouseColumn column) { * @return value object with default value, either null or empty */ public static ClickHouseValue newValue(ClickHouseDataType type) { - return newValue(type, null); + return newValue(ClickHouseChecker.nonNull(type, "type"), null); } private static ClickHouseValue newValue(ClickHouseDataType type, ClickHouseColumn column) { - ClickHouseValue value; - switch (ClickHouseChecker.nonNull(type, "type")) { // still faster than EnumMap and with less overhead - case Enum: - case Enum8: - case Int8: - value = ClickHouseByteValue.ofNull(); - break; - case UInt8: - case Enum16: - case Int16: - value = ClickHouseShortValue.ofNull(); - break; - case UInt16: - case Int32: - value = ClickHouseIntegerValue.ofNull(); - break; - case UInt32: - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Int64: - value = ClickHouseLongValue.ofNull(false); - break; - case UInt64: - value = ClickHouseLongValue.ofNull(true); - break; - case Int128: - case UInt128: - case Int256: - case UInt256: - value = ClickHouseBigIntegerValue.ofNull(); - break; - case Float32: - value = ClickHouseFloatValue.ofNull(); - break; - case Float64: - value = ClickHouseDoubleValue.ofNull(); - break; - case Decimal: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal256: - value = ClickHouseBigDecimalValue.ofNull(); - break; - case Date: - case Date32: - value = ClickHouseDateValue.ofNull(); - break; - case DateTime: - case DateTime32: - case DateTime64: - value = ClickHouseDateTimeValue.ofNull(column == null ? 0 : column.getScale()); - break; - case IPv4: - value = ClickHouseIpv4Value.ofNull(); - break; - case IPv6: - value = ClickHouseIpv6Value.ofNull(); - break; - case FixedString: - case String: - value = ClickHouseStringValue.ofNull(); - break; - case UUID: - value = ClickHouseUuidValue.ofNull(); - break; - case Point: - value = ClickHouseGeoPointValue.ofOrigin(); - break; - case Ring: - value = ClickHouseGeoRingValue.ofEmpty(); - break; - case Polygon: - value = ClickHouseGeoPolygonValue.ofEmpty(); - break; - case MultiPolygon: - value = ClickHouseGeoMultiPolygonValue.ofEmpty(); - break; - case Array: - value = ClickHouseArrayValue.ofEmpty(); - break; - case Map: - if (column == null) { - throw new IllegalArgumentException("column types for key and value are required"); + ClickHouseValue value = null; + switch (type) { // still faster than EnumMap and with less overhead + case Enum: + case Enum8: + case Int8: + value = ClickHouseByteValue.ofNull(); + break; + case UInt8: + case Enum16: + case Int16: + value = ClickHouseShortValue.ofNull(); + break; + case UInt16: + case Int32: + value = ClickHouseIntegerValue.ofNull(); + break; + case UInt32: + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Int64: + value = ClickHouseLongValue.ofNull(false); + break; + case UInt64: + value = ClickHouseLongValue.ofNull(true); + break; + case Int128: + case UInt128: + case Int256: + case UInt256: + value = ClickHouseBigIntegerValue.ofNull(); + break; + case Float32: + value = ClickHouseFloatValue.ofNull(); + break; + case Float64: + value = ClickHouseDoubleValue.ofNull(); + break; + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + value = ClickHouseBigDecimalValue.ofNull(); + break; + case Date: + case Date32: + value = ClickHouseDateValue.ofNull(); + break; + case DateTime: + case DateTime32: + case DateTime64: { + if (column == null) { + value = ClickHouseDateTimeValue.ofNull(0); + } else if (column.getTimeZone() == null) { + value = ClickHouseDateTimeValue.ofNull(column.getScale()); + } else { + value = ClickHouseOffsetDateTimeValue.ofNull(column.getScale(), column.getTimeZone()); + } + break; + } + case IPv4: + value = ClickHouseIpv4Value.ofNull(); + break; + case IPv6: + value = ClickHouseIpv6Value.ofNull(); + break; + case FixedString: + case String: + value = ClickHouseStringValue.ofNull(); + break; + case UUID: + value = ClickHouseUuidValue.ofNull(); + break; + case Point: + value = ClickHouseGeoPointValue.ofOrigin(); + break; + case Ring: + value = ClickHouseGeoRingValue.ofEmpty(); + break; + case Polygon: + value = ClickHouseGeoPolygonValue.ofEmpty(); + break; + case MultiPolygon: + value = ClickHouseGeoMultiPolygonValue.ofEmpty(); + break; + case AggregateFunction: + if (column != null) { + if (column.getAggregateFunction() == ClickHouseAggregateFunction.groupBitmap) { + value = ClickHouseBitmapValue.ofEmpty(column.getNestedColumns().get(0).getDataType()); } - value = ClickHouseMapValue.ofEmpty(column.getKeyInfo().getDataType().getJavaClass(), - column.getValueInfo().getDataType().getJavaClass()); - break; - case Nested: - if (column == null) { - throw new IllegalArgumentException("nested column types are required"); + } + break; + case Array: + if (column == null) { + value = ClickHouseArrayValue.ofEmpty(); + } else if (column.getArrayNestedLevel() > 1) { + value = ClickHouseArrayValue.of( + (Object[]) createPrimitiveArray(column.getArrayBaseColumn().getDataType().getPrimitiveClass(), + 0, column.getArrayNestedLevel())); + } else { + Class javaClass = column.getArrayBaseColumn().getDataType().getPrimitiveClass(); + if (byte.class == javaClass) { + value = ClickHouseByteArrayValue.ofEmpty(); + } else if (short.class == javaClass) { + value = ClickHouseShortArrayValue.ofEmpty(); + } else if (int.class == javaClass) { + value = ClickHouseIntArrayValue.ofEmpty(); + } else if (long.class == javaClass) { + value = ClickHouseLongArrayValue.ofEmpty(); + } else if (float.class == javaClass) { + value = ClickHouseFloatArrayValue.ofEmpty(); + } else if (double.class == javaClass) { + value = ClickHouseDoubleArrayValue.ofEmpty(); + } else { + value = ClickHouseArrayValue.ofEmpty(); } - value = ClickHouseNestedValue.ofEmpty(column.getNestedColumns()); - break; - case Tuple: - value = ClickHouseTupleValue.of(); - break; - default: - throw new IllegalArgumentException("Unsupported data type: " + type.name()); + } + break; + case Map: + if (column == null) { + throw new IllegalArgumentException("column types for key and value are required"); + } + value = ClickHouseMapValue.ofEmpty(column.getKeyInfo().getDataType().getObjectClass(), + column.getValueInfo().getDataType().getObjectClass()); + break; + case Nested: + if (column == null) { + throw new IllegalArgumentException("nested column types are required"); + } + value = ClickHouseNestedValue.ofEmpty(column.getNestedColumns()); + break; + case Tuple: + value = ClickHouseTupleValue.of(); + break; + case Nothing: + value = ClickHouseEmptyValue.INSTANCE; + break; + default: + break; + } + + if (value == null) { + throw new IllegalArgumentException("Unsupported data type: " + type.name()); } return value; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseVersion.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseVersion.java index 4c92d74d2..d2d603480 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseVersion.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseVersion.java @@ -134,6 +134,55 @@ protected ClickHouseVersion(boolean latest, int year, int feature, int maintenan } } + /** + * Compares current version and the given one. When {@code includeEmptyParts} is + * {@code true}, this method returns 0(instead of 1) when comparing '21.3.1.2' + * with '21.3', because they're in the same series of '21.3'. + * + * @param o the object to be compared + * @param sameSeriesComparison whether compare if two version are in same series + * @return a negative integer, zero, or a positive integer as this object is + * less than, equal to, or greater than the specified object + */ + protected int compareTo(ClickHouseVersion o, boolean sameSeriesComparison) { + if (o == null) { + o = defaultVersion; + } + + if (this == o) { + return 0; + } + + if (latest) { + return o.latest ? 0 : 1; + } else if (o.latest) { + return -1; + } + + int result = year - o.year; + if (result != 0) { + return result; + } else if (sameSeriesComparison && o.feature == 0 && o.maintenance == 0 && o.build == 0) { + return 0; + } + + result = feature - o.feature; + if (result != 0) { + return result; + } else if (sameSeriesComparison && o.maintenance == 0 && o.build == 0) { + return 0; + } + + result = maintenance - o.maintenance; + if (result != 0) { + return result; + } else if (sameSeriesComparison && o.build == 0) { + return 0; + } + + return build - o.build; + } + /** * Checks if the version belongs to the given series. For example: 21.3.1.1 * belongs to 21.3 series but not 21.4 or 21.3.2. @@ -266,33 +315,7 @@ public int getPatch() { * otherwise */ public boolean isNewerOrEqualTo(ClickHouseVersion version) { - if (version == null) { - version = defaultVersion; - } - - if (isLatest()) { - return true; - } else if (version.isLatest()) { - return false; - } - - if (year < version.year) { - return false; - } - - if (version.feature == 0 && version.maintenance == 0 && version.build == 0) { - return true; - } else if (feature < version.feature) { - return false; - } - - if (version.maintenance == 0 && version.build == 0) { - return true; - } else if (maintenance < version.maintenance) { - return false; - } - - return version.build == 0 || build >= version.build; + return compareTo(version, true) >= 0; } /** @@ -311,18 +334,18 @@ public boolean isNewerOrEqualTo(String version) { /** * Checks if the version is newer than the given one. Same as - * {@code compareTo(version) >= 0}. + * {@code compareTo(version) > 0}. * * @param version version to compare * @return true if the version is newer than the given one; false otherwise */ public boolean isNewerThan(ClickHouseVersion version) { - return compareTo(version) > 0; + return compareTo(version, false) > 0; } /** * Checks if the version is newer than the given one. Same as - * {@code compareTo(version) >= 0}. + * {@code compareTo(version) > 0}. * * @param version version to compare * @return true if the version is newer than the given one; false otherwise @@ -342,34 +365,7 @@ public boolean isNewerThan(String version) { * otherwise */ public boolean isOlderOrEqualTo(ClickHouseVersion version) { - if (version == null) { - version = defaultVersion; - } - - // latest version is NOT older than latest - if (isLatest()) { - return false; - } else if (version.isLatest()) { - return true; - } - - if (year > version.year) { - return false; - } - - if (version.feature == 0 && version.maintenance == 0 && version.build == 0) { - return true; - } else if (feature > version.feature) { - return false; - } - - if (version.maintenance == 0 && version.build == 0) { - return true; - } else if (maintenance > version.maintenance) { - return false; - } - - return version.build == 0 || build <= version.build; + return compareTo(version, true) <= 0; } /** @@ -388,18 +384,18 @@ public boolean isOlderOrEqualTo(String version) { /** * Checks if the version is older than the given one. Same as - * {@code compareTo(version) <= 0}. + * {@code compareTo(version) < 0}. * * @param version version to compare * @return true if the version is older than the given one; false otherwise */ public boolean isOlderThan(ClickHouseVersion version) { - return compareTo(version) < 0; + return compareTo(version, false) < 0; } /** * Checks if the version is older than the given one. Same as - * {@code compareTo(version) <= 0}. + * {@code compareTo(version) < 0}. * * @param version version to compare * @return true if the version is older than the given one; false otherwise @@ -488,36 +484,7 @@ public boolean check(String range) { @Override public int compareTo(ClickHouseVersion o) { - if (o == null) { - o = defaultVersion; - } - - if (this == o) { - return 0; - } - - if (latest) { - return o.latest ? 0 : 1; - } else if (o.latest) { - return -1; - } - - int result = year - o.year; - if (result != 0) { - return result; - } - - result = feature - o.feature; - if (result != 0) { - return result; - } - - result = maintenance - o.maintenance; - if (result != 0) { - return result; - } - - return build - o.build; + return compareTo(o, false); } @Override diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java index 94f14c9fa..6cc43f8cb 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java @@ -1,11 +1,18 @@ package com.clickhouse.client.config; +import java.io.Serializable; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseCompression; +import com.clickhouse.client.ClickHouseFormat; /** * Generic client options. */ -public enum ClickHouseClientOption implements ClickHouseConfigOption { +public enum ClickHouseClientOption implements ClickHouseOption { /** * Whether the client should run in async mode(e.g. * {@link com.clickhouse.client.ClickHouseClient#execute(com.clickhouse.client.ClickHouseRequest)} @@ -17,12 +24,34 @@ public enum ClickHouseClientOption implements ClickHouseConfigOption { */ CLIENT_NAME("client_name", "ClickHouse Java Client", "Client name, which is either 'client_name' or 'http_user_agent' shows up in system.query_log table."), + + /** + * Whether server will compress response to client or not. + */ + COMPRESS("compress", true, "Whether the server will compress response it sends to client."), + /** + * Whether server will decompress request from client or not. + */ + DECOMPRESS("decompress", false, "Whether the server will decompress request from client."), + /** + * Compression algorithm server will use to compress response, when + * {@link #COMPRESS} is {@code true}. + */ + COMPRESS_ALGORITHM("compress_alogrithm", ClickHouseCompression.LZ4, "Algorithm used for compressing response."), + /** + * Compression algorithm server will use to decompress request, when + * {@link #DECOMPRESS} is {@code true}. + */ + DECOMPRESS_ALGORITHM("decompress_alogrithm", ClickHouseCompression.GZIP, "Algorithm for decompressing request."), /** - * Case-insensitive transport level compression algorithm. See - * {@link com.clickhouse.client.ClickHouseCompression} for all possible options. + * Compression level for compressing server response. */ - COMPRESSION("compression", "LZ4", - "Transport level compression algorithm used when exchanging data between server and client."), + COMPRESS_LEVEL("compress_level", 3, "Compression level for response, from 0 to 9(low to high)"), + /** + * Compression level for decompress client request. + */ + DECOMPRESS_LEVEL("decompress_level", 3, "Compression level for request, from 0 to 9(low to high)"), + /** * Connection timeout in milliseconds. */ @@ -35,15 +64,26 @@ public enum ClickHouseClientOption implements ClickHouseConfigOption { /** * Default format. */ - FORMAT("format", "TabSeparatedWithNamesAndTypes", "Default format."), + FORMAT("format", ClickHouseFormat.TabSeparated, "Default format."), + /** + * Whether to log leading comment(as log_comment in system.query_log) of the + * query. + */ + LOG_LEADING_COMMENT("log_leading_comment", false, + "Whether to log leading comment(as log_comment in system.query_log) of the query."), /** * Maximum buffer size in byte used for streaming. */ MAX_BUFFER_SIZE("max_buffer_size", 8 * 1024, "Maximum buffer size in byte used for streaming."), + /** + * Maximum comression block size in byte, only useful when {@link #DECOMPRESS} + * is {@code true}. + */ + MAX_COMPRESS_BLOCK_SIZE("max_compress_block_size", 1024 * 1024, "Maximum comression block size in byte."), /** * Maximum query execution time in seconds. */ - MAX_EXECUTION_TIME("max_execution_time", 0, "Maximum query execution time in seconds."), + MAX_EXECUTION_TIME("max_execution_time", 0, "Maximum query execution time in seconds, 0 means no limit."), /** * Maximum queued in-memory buffers. */ @@ -63,58 +103,69 @@ public enum ClickHouseClientOption implements ClickHouseConfigOption { /** * Maximum size of thread pool for each client. */ - MAX_THREADS_PER_CLIENT("max_threads_per_client", 1, + MAX_THREADS_PER_CLIENT("max_threads_per_client", 0, "Size of thread pool for each client instance, 0 or negative number means the client will use shared thread pool."), /** * Whether to enable retry. */ RETRY("retry", true, "Whether to retry when there's connection issue."), /** - * Whether to reuse wrapper of value. + * Whether to reuse wrapper of value(e.g. ClickHouseValue or + * ClickHouseRecord) for memory efficiency. */ - REUSE_VALUE_WRAPPER("reuse_value_wrapper", true, "Whether to reuse value-wrapper for memory efficiency."), + REUSE_VALUE_WRAPPER("reuse_value_wrapper", true, + "Whether to reuse wrapper of value(e.g. ClickHouseValue or ClickHouseRecord) for memory efficiency."), /** - * Socket timeout in milliseconds. + * Server timezone. */ - SOCKET_TIMEOUT("socket_timeout", 30 * 1000, "Socket timeout in milliseconds."), + SERVER_TIME_ZONE("server_time_zone", "", "Server timezone."), + /** + * Server version. + */ + SERVER_VERSION("server_version", "", "Server version."), /** * Whether to check if session id is validate. */ - SESSION_CHECK("session_check", false, "Whether to check if session id is validate."), + SESSION_CHECK("session_check", false, "Whether to check if existence of session id."), /** * Session timeout in milliseconds. */ SESSION_TIMEOUT("session_timeout", 0, "Session timeout in milliseconds. 0 or negative number means same as server default."), + /** + * Socket timeout in milliseconds. + */ + SOCKET_TIMEOUT("socket_timeout", 30 * 1000, "Socket timeout in milliseconds."), /** * Whether to enable SSL for the connection. */ - SSL("ssl", false, "enable SSL/TLS for the connection"), + SSL("ssl", false, "Whether to enable SSL/TLS for the connection."), /** * SSL mode. */ - SSL_MODE("sslmode", "strict", "verify or not certificate: none (don't verify), strict (verify)"), + SSL_MODE("sslmode", ClickHouseSslMode.STRICT, "verify or not certificate: none (don't verify), strict (verify)"), /** * SSL root certificiate. */ - SSL_ROOT_CERTIFICATE("sslrootcert", "", "SSL/TLS root certificate"), + SSL_ROOT_CERTIFICATE("sslrootcert", "", "SSL/TLS root certificate."), /** * SSL certificiate. */ - SSL_CERTIFICATE("sslcert", "", "SSL/TLS certificate"), + SSL_CERTIFICATE("sslcert", "", "SSL/TLS certificate."), /** * SSL key. */ - SSL_KEY("sslkey", "", "SSL/TLS key"), + SSL_KEY("sslkey", "", "SSL/TLS key."), /** * Whether to use objects in array or not. */ - USE_OBJECTS_IN_ARRAYS("use_objects_in_arrays", false, "Whether Object[] should be used instead primitive arrays."), + USE_OBJECTS_IN_ARRAYS("use_objects_in_arrays", false, + "Whether Object[] should be used instead of primitive arrays."), /** * Whether to use server time zone. */ USE_SERVER_TIME_ZONE("use_server_time_zone", true, - "Whether to use time zone from server. On connection init select timezone() will be executed"), + "Whether to use server time zone. On connection init select timezone() will be executed"), /** * Whether to use time zone from server for Date. */ @@ -126,13 +177,37 @@ public enum ClickHouseClientOption implements ClickHouseConfigOption { * Custom time zone. Only works when {@code use_server_time_zone} is set to * false. */ - USE_TIME_ZONE("use_time_zone", "", "Which time zone to use"); + USE_TIME_ZONE("use_time_zone", "", "Which time zone to use. Only works when use_server_time_zone is false."); private final String key; - private final Object defaultValue; - private final Class clazz; + private final Serializable defaultValue; + private final Class clazz; private final String description; + private static final Map options; + + static { + Map map = new HashMap<>(); + + for (ClickHouseClientOption o : values()) { + if (map.put(o.getKey(), o) != null) { + throw new IllegalStateException("Duplicated key found: " + o.getKey()); + } + } + + options = Collections.unmodifiableMap(map); + } + + /** + * Gets client option by key. + * + * @param key key of the option + * @return client option object, or null if not found + */ + public static ClickHouseClientOption fromKey(String key) { + return options.get(key); + } + /** * Constructor of an option for client. * @@ -143,7 +218,7 @@ public enum ClickHouseClientOption implements ClickHouseConfigOption { * @param defaultValue non-null default value * @param description non-null description of this option */ - ClickHouseClientOption(String key, T defaultValue, String description) { + ClickHouseClientOption(String key, T defaultValue, String description) { this.key = ClickHouseChecker.nonNull(key, "key"); this.defaultValue = ClickHouseChecker.nonNull(defaultValue, "defaultValue"); this.clazz = defaultValue.getClass(); @@ -151,7 +226,7 @@ ClickHouseClientOption(String key, T defaultValue, String description) { } @Override - public Object getDefaultValue() { + public Serializable getDefaultValue() { return defaultValue; } @@ -166,7 +241,7 @@ public String getKey() { } @Override - public Class getValueType() { + public Class getValueType() { return clazz; } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaults.java b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaults.java index 01c1058ea..1e47c0d4a 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaults.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaults.java @@ -1,6 +1,11 @@ package com.clickhouse.client.config; +import java.io.Serializable; + import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseCompression; +import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.client.ClickHouseProtocol; /** * System-wide default options. System properties and environment variables can @@ -12,7 +17,7 @@ * {@code -Ddefault_async=false} on the Java command line, or setting * environment variable {@code DEFAULT_ASYNC=false}. */ -public enum ClickHouseDefaults implements ClickHouseConfigOption { +public enum ClickHouseDefaults implements ClickHouseOption { /** * Default execution mode. */ @@ -32,7 +37,7 @@ public enum ClickHouseDefaults implements ClickHouseConfigOption { /** * Default protocol. */ - PROTOCOL("protocol", "ANY", "Protocol to use."), + PROTOCOL("protocol", ClickHouseProtocol.ANY, "Protocol to use."), /** * Default server port. */ @@ -53,14 +58,10 @@ public enum ClickHouseDefaults implements ClickHouseConfigOption { * Default password. */ PASSWORD("password", "", "Password for authentication."), - /** - * Default compression. - */ - COMPRESSION("compression", "LZ4", "Preferred compression alogrithm used in data transferring."), /** * Default format. */ - FORMAT("format", "TabSeparated", "Preferred data format for serialization and deserialization."), + FORMAT("format", ClickHouseFormat.TabSeparated, "Preferred data format for serialization and deserialization."), /** * Max threads. */ @@ -69,6 +70,14 @@ public enum ClickHouseDefaults implements ClickHouseConfigOption { * Max requests. */ MAX_REQUESTS("max_requests", 0, "Maximum size of shared thread pool, 0 means no limit."), + /** + * Server time zone, defaults to {@code UTC}. + */ + SERVER_TIME_ZONE("time_zone", "UTC", "Server time zone."), + /** + * Server version, defaults to {@code latest}. + */ + SERVER_VERSION("version", "latest", "Server version"), /** * Whether to resolve DNS SRV name using * {@link com.clickhouse.client.naming.SrvResolver}(e.g. resolve SRV record to @@ -77,11 +86,11 @@ public enum ClickHouseDefaults implements ClickHouseConfigOption { SRV_RESOLVE("srv_resolve", false, "Whether to resolve DNS SRV name."); private final String key; - private final Object defaultValue; + private final Serializable defaultValue; private final Class clazz; private final String description; - ClickHouseDefaults(String key, T defaultValue, String description) { + ClickHouseDefaults(String key, T defaultValue, String description) { this.key = ClickHouseChecker.nonNull(key, "key"); this.defaultValue = ClickHouseChecker.nonNull(defaultValue, "defaultValue"); this.clazz = defaultValue.getClass(); @@ -89,7 +98,7 @@ ClickHouseDefaults(String key, T defaultValue, String description) { } @Override - public Object getDefaultValue() { + public Serializable getDefaultValue() { return defaultValue; } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseConfigOption.java b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseOption.java similarity index 54% rename from clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseConfigOption.java rename to clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseOption.java index ef9750aa3..a0eeffc2a 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseConfigOption.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseOption.java @@ -1,5 +1,6 @@ package com.clickhouse.client.config; +import java.io.Serializable; import java.util.Optional; /** @@ -7,26 +8,22 @@ * composed of key, default value(which implies type of the value) and * description. */ -public interface ClickHouseConfigOption { +public interface ClickHouseOption extends Serializable { /** * Converts given string to a typed value. * * @param type of the value * @param value value in string format - * @param clazz class of the value + * @param clazz non-null class of the value * @return typed value */ - static T fromString(String value, Class clazz) { + @SuppressWarnings("unchecked") + static T fromString(String value, Class clazz) { if (value == null || clazz == null) { throw new IllegalArgumentException("Non-null value and class are required"); } - if (clazz == int.class || clazz == Integer.class) { - return clazz.cast(Integer.valueOf(value)); - } - if (clazz == long.class || clazz == Long.class) { - return clazz.cast(Long.valueOf(value)); - } + T result; if (clazz == boolean.class || clazz == Boolean.class) { final Boolean boolValue; if ("1".equals(value) || "0".equals(value)) { @@ -34,10 +31,25 @@ static T fromString(String value, Class clazz) { } else { boolValue = Boolean.valueOf(value); } - return clazz.cast(boolValue); + result = clazz.cast(boolValue); + } else if (byte.class == clazz || Byte.class == clazz) { + result = clazz.cast(value.isEmpty() ? Byte.valueOf((byte) 0) : Byte.valueOf(value)); + } else if (short.class == clazz || Short.class == clazz) { + result = clazz.cast(value.isEmpty() ? Short.valueOf((short) 0) : Short.valueOf(value)); + } else if (int.class == clazz || Integer.class == clazz) { + result = clazz.cast(value.isEmpty() ? Integer.valueOf(0) : Integer.valueOf(value)); + } else if (long.class == clazz || Long.class == clazz) { + result = clazz.cast(value.isEmpty() ? Long.valueOf(0L) : Long.valueOf(value)); + } else if (float.class == clazz || Float.class == clazz) { + result = clazz.cast(value.isEmpty() ? Float.valueOf(0F) : Float.valueOf(value)); + } else if (double.class == clazz || Double.class == clazz) { + result = clazz.cast(value.isEmpty() ? Double.valueOf(0D) : Double.valueOf(value)); + } else if (Enum.class.isAssignableFrom(clazz)) { + result = (T) Enum.valueOf((Class) clazz, value); + } else { + result = clazz.cast(value); } - - return clazz.cast(value); + return result; } /** @@ -45,35 +57,46 @@ static T fromString(String value, Class clazz) { * * @return default value of the option */ - Object getDefaultValue(); + Serializable getDefaultValue(); /** - * Gets default value from environment variable. By default the environment - * variable is named as {@link #getPrefix()} + "_" + {@link #name()} in upper - * case. + * Gets trimmed default value from environment variable. By default the + * environment variable is named as {@link #getPrefix()} + "_" + {@link #name()} + * in upper case. * - * @return default value defined in environment variable + * @return trimmed default value defined in environment variable */ default Optional getDefaultValueFromEnvVar() { String prefix = getPrefix().toUpperCase(); String optionName = name(); int length = optionName.length(); - return Optional.ofNullable(System.getenv(new StringBuilder(length + prefix.length() + 1).append(prefix) - .append('_').append(optionName.toUpperCase()).toString())); + + String value = System.getenv(new StringBuilder(length + prefix.length() + 1).append(prefix).append('_') + .append(optionName.toUpperCase()).toString()); + if (value != null) { + value = value.trim(); + } + return Optional.ofNullable(value); } /** - * Gets default value from system property. By default the system property is - * named as {@link #getPrefix()} + "_" + {@link #name()} in lower case. + * Gets trimmed default value from system property. By default the system + * property is named as {@link #getPrefix()} + "_" + {@link #name()} in lower + * case. * - * @return default value defined in system property + * @return trimmed default value defined in system property */ default Optional getDefaultValueFromSysProp() { String prefix = getPrefix().toLowerCase(); String optionName = name(); int length = optionName.length(); - return Optional.ofNullable(System.getProperty(new StringBuilder(length + prefix.length() + 1).append(prefix) - .append('_').append(optionName.toLowerCase()).toString())); + + String value = System.getProperty(new StringBuilder(length + prefix.length() + 1).append(prefix).append('_') + .append(optionName.toLowerCase()).toString()); + if (value != null) { + value = value.trim(); + } + return Optional.ofNullable(value); } /** @@ -90,7 +113,8 @@ default Optional getDefaultValueFromSysProp() { * * @return effective default value */ - default Object getEffectiveDefaultValue() { + @SuppressWarnings("unchecked") + default Serializable getEffectiveDefaultValue() { Optional value = getDefaultValueFromEnvVar(); if (!value.isPresent() || value.get().isEmpty()) { @@ -101,7 +125,7 @@ default Object getEffectiveDefaultValue() { return getDefaultValue(); } - return fromString(value.get(), getValueType()); + return fromString(value.get(), (Class) getValueType()); } /** diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/BinaryStreamUtils.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/BinaryStreamUtils.java index 508917ed4..8858cef5b 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/BinaryStreamUtils.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/BinaryStreamUtils.java @@ -4,20 +4,25 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.Reader; import java.math.BigDecimal; import java.math.BigInteger; import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; import java.nio.ByteBuffer; +import java.nio.channels.Channels; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneOffset; +import java.util.TimeZone; import java.util.concurrent.TimeUnit; import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseDataType; import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.ClickHouseValues; @@ -166,20 +171,20 @@ public static byte readByte(InputStream input) throws IOException { } /** - * Read {@code size} bytes from given input stream. It behaves in the same way - * as {@link java.io.DataInput#readFully(byte[])}. + * Reads {@code length} bytes from given input stream. It behaves in the same + * way as {@link java.io.DataInput#readFully(byte[])}. * - * @param input input stream - * @param size number of bytes to read - * @return byte array and its length should be {@code size} + * @param input non-null input stream + * @param length number of bytes to read + * @return byte array and its length should be {@code length} * @throws IOException when failed to read value from input stream, not able to * retrieve all bytes, or reached end of the stream */ - public static byte[] readBytes(InputStream input, int size) throws IOException { + public static byte[] readBytes(InputStream input, int length) throws IOException { int count = 0; - byte[] bytes = new byte[size]; - while (count < size) { - int n = input.read(bytes, count, size - count); + byte[] bytes = new byte[length]; + while (count < length) { + int n = input.read(bytes, count, length - count); if (n < 0) { try { input.close(); @@ -189,7 +194,7 @@ public static byte[] readBytes(InputStream input, int size) throws IOException { throw count == 0 ? new EOFException() : new IOException(ClickHouseUtils - .format("Reached end of input stream after reading %d of %d bytes", count, size)); + .format("Reached end of input stream after reading %d of %d bytes", count, length)); } count += n; } @@ -197,6 +202,89 @@ public static byte[] readBytes(InputStream input, int size) throws IOException { return bytes; } + /** + * Writes bytes into given output stream. + * + * @param output non-null output stream + * @param buffer non-null byte buffer + * @throws IOException when failed to write value to output stream or reached + * end of the stream + */ + @SuppressWarnings("squid:S2095") + public static void writeByteBuffer(OutputStream output, ByteBuffer buffer) throws IOException { + Channels.newChannel(output).write(buffer); + } + + /** + * Reads bitmap from given input stream. It behaves in a similar way as + * {@link java.io.DataInput#readFully(byte[])}. + * + * @param input non-null input + * @param dataType number of characters to read + * @return non-null bitmap wrapper class + * @throws IOException when failed to read value from input stream, not able to + * retrieve all bytes, or reached end of the stream + */ + public static ClickHouseBitmap readBitmap(InputStream input, ClickHouseDataType dataType) throws IOException { + return ClickHouseBitmap.deserialize(input, dataType); + } + + /** + * Writes bitmap into given output stream. + * + * @param output non-null output stream + * @param bitmap non-null bitmap + * @throws IOException when failed to write value to output stream or reached + * end of the stream + */ + public static void writeBitmap(OutputStream output, ClickHouseBitmap bitmap) throws IOException { + writeByteBuffer(output, bitmap.toByteBuffer()); + } + + /** + * Writes bytes into given output stream. + * + * @param output non-null output stream + * @param bytes non-null byte array + * @throws IOException when failed to write value to output stream or reached + * end of the stream + */ + public static void writeBytes(OutputStream output, byte[] bytes) throws IOException { + output.write(bytes); + } + + /** + * Reads {@code length} characters from given reader. It behaves in a similar + * way as {@link java.io.DataInput#readFully(byte[])}. + * + * @param input non-null reader + * @param length number of characters to read + * @return character array and its length should be {@code length} + * @throws IOException when failed to read value from input stream, not able to + * retrieve all bytes, or reached end of the stream + */ + public static char[] readCharacters(Reader input, int length) throws IOException { + int count = 0; + char[] chars = new char[length]; + while (count < length) { + int n = input.read(chars, count, length - count); + if (n < 0) { + try { + input.close(); + } catch (IOException e) { + // ignore + } + + throw count == 0 ? new EOFException() + : new IOException(ClickHouseUtils + .format("Reached end of reader after reading %d of %d characters", count, length)); + } + count += n; + } + + return chars; + } + /** * Read boolean from given input stream. It uses {@link #readByte(InputStream)} * to get value and return {@code true} only when the value is {@code 1}. @@ -1024,11 +1112,11 @@ public static void writeBigInteger(OutputStream output, BigInteger value, int le public static BigDecimal readDecimal(InputStream input, int precision, int scale) throws IOException { BigDecimal v; - if (precision <= 9) { + if (precision <= ClickHouseDataType.Decimal32.getMaxScale()) { v = readDecimal32(input, scale); - } else if (precision <= 18) { + } else if (precision <= ClickHouseDataType.Decimal64.getMaxScale()) { v = readDecimal64(input, scale); - } else if (precision <= 38) { + } else if (precision <= ClickHouseDataType.Decimal128.getMaxScale()) { v = readDecimal128(input, scale); } else { v = readDecimal256(input, scale); @@ -1050,11 +1138,11 @@ public static BigDecimal readDecimal(InputStream input, int precision, int scale */ public static void writeDecimal(OutputStream output, BigDecimal value, int precision, int scale) throws IOException { - if (precision > 38) { + if (precision > ClickHouseDataType.Decimal128.getMaxScale()) { writeDecimal256(output, value, scale); - } else if (precision > 18) { + } else if (precision > ClickHouseDataType.Decimal64.getMaxScale()) { writeDecimal128(output, value, scale); - } else if (precision > 9) { + } else if (precision > ClickHouseDataType.Decimal32.getMaxScale()) { writeDecimal64(output, value, scale); } else { writeDecimal32(output, value, scale); @@ -1071,8 +1159,8 @@ public static void writeDecimal(OutputStream output, BigDecimal value, int preci * end of the stream */ public static BigDecimal readDecimal32(InputStream input, int scale) throws IOException { - return BigDecimal.valueOf(readInt32(input), - ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 9)); + return BigDecimal.valueOf(readInt32(input), ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, + ClickHouseDataType.Decimal32.getMaxScale())); } /** @@ -1088,8 +1176,8 @@ public static BigDecimal readDecimal32(InputStream input, int scale) throws IOEx public static void writeDecimal32(OutputStream output, BigDecimal value, int scale) throws IOException { writeInt32(output, ClickHouseChecker.between( - value.multiply(BigDecimal.TEN - .pow(ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 9))), + value.multiply(BigDecimal.TEN.pow(ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, + 0, ClickHouseDataType.Decimal32.getMaxScale()))), ClickHouseValues.TYPE_BIG_DECIMAL, DECIMAL32_MIN, DECIMAL32_MAX).intValue()); } @@ -1103,8 +1191,8 @@ public static void writeDecimal32(OutputStream output, BigDecimal value, int sca * end of the stream */ public static BigDecimal readDecimal64(InputStream input, int scale) throws IOException { - return BigDecimal.valueOf(readInt64(input), - ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 18)); + return BigDecimal.valueOf(readInt64(input), ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, + ClickHouseDataType.Decimal64.getMaxScale())); } /** @@ -1120,8 +1208,9 @@ public static BigDecimal readDecimal64(InputStream input, int scale) throws IOEx public static void writeDecimal64(OutputStream output, BigDecimal value, int scale) throws IOException { writeInt64(output, ClickHouseChecker.between( - ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 18) == 0 ? value - : value.multiply(BigDecimal.TEN.pow(scale)), + ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, + ClickHouseDataType.Decimal64.getMaxScale()) == 0 ? value + : value.multiply(BigDecimal.TEN.pow(scale)), ClickHouseValues.TYPE_BIG_DECIMAL, DECIMAL64_MIN, DECIMAL64_MAX).longValue()); } @@ -1135,7 +1224,8 @@ public static void writeDecimal64(OutputStream output, BigDecimal value, int sca * end of the stream */ public static BigDecimal readDecimal128(InputStream input, int scale) throws IOException { - return new BigDecimal(readInt128(input), ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 38)); + return new BigDecimal(readInt128(input), ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, + ClickHouseDataType.Decimal128.getMaxScale())); } /** @@ -1151,8 +1241,9 @@ public static BigDecimal readDecimal128(InputStream input, int scale) throws IOE public static void writeDecimal128(OutputStream output, BigDecimal value, int scale) throws IOException { writeInt128(output, ClickHouseChecker.between( - ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 38) == 0 ? value - : value.multiply(BigDecimal.TEN.pow(scale)), + ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, + ClickHouseDataType.Decimal128.getMaxScale()) == 0 ? value + : value.multiply(BigDecimal.TEN.pow(scale)), ClickHouseValues.TYPE_BIG_DECIMAL, DECIMAL128_MIN, DECIMAL128_MAX).toBigInteger()); } @@ -1166,7 +1257,8 @@ public static void writeDecimal128(OutputStream output, BigDecimal value, int sc * end of the stream */ public static BigDecimal readDecimal256(InputStream input, int scale) throws IOException { - return new BigDecimal(readInt256(input), ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 76)); + return new BigDecimal(readInt256(input), ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, + ClickHouseDataType.Decimal256.getMaxScale())); } /** @@ -1182,8 +1274,9 @@ public static BigDecimal readDecimal256(InputStream input, int scale) throws IOE public static void writeDecimal256(OutputStream output, BigDecimal value, int scale) throws IOException { writeInt256(output, ClickHouseChecker.between( - ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 76) == 0 ? value - : value.multiply(BigDecimal.TEN.pow(scale)), + ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, + ClickHouseDataType.Decimal256.getMaxScale()) == 0 ? value + : value.multiply(BigDecimal.TEN.pow(scale)), ClickHouseValues.TYPE_BIG_DECIMAL, DECIMAL256_MIN, DECIMAL256_MAX).toBigInteger()); } @@ -1191,11 +1284,12 @@ public static void writeDecimal256(OutputStream output, BigDecimal value, int sc * Read {@link java.time.LocalDate} from given input stream. * * @param input non-null input stream + * @param tz time zone, null is treated as UTC * @return local date * @throws IOException when failed to read value from input stream or reached * end of the stream */ - public static LocalDate readDate(InputStream input) throws IOException { + public static LocalDate readDate(InputStream input, TimeZone tz) throws IOException { return LocalDate.ofEpochDay(readUnsignedInt16(input)); } @@ -1204,10 +1298,11 @@ public static LocalDate readDate(InputStream input) throws IOException { * * @param output non-null output stream * @param value local date + * @param tz time zone, null is treated as UTC * @throws IOException when failed to write value to output stream or reached * end of the stream */ - public static void writeDate(OutputStream output, LocalDate value) throws IOException { + public static void writeDate(OutputStream output, LocalDate value, TimeZone tz) throws IOException { int days = (int) value.toEpochDay(); writeUnsignedInt16(output, ClickHouseChecker.between(days, ClickHouseValues.TYPE_DATE, 0, U_INT16_MAX)); } @@ -1216,11 +1311,12 @@ public static void writeDate(OutputStream output, LocalDate value) throws IOExce * Read {@link java.time.LocalDate} from given input stream. * * @param input non-null input stream + * @param tz time zone, null is treated as UTC * @return local date * @throws IOException when failed to read value from input stream or reached * end of the stream */ - public static LocalDate readDate32(InputStream input) throws IOException { + public static LocalDate readDate32(InputStream input, TimeZone tz) throws IOException { return LocalDate.ofEpochDay(readInt32(input)); } @@ -1229,10 +1325,11 @@ public static LocalDate readDate32(InputStream input) throws IOException { * * @param output non-null output stream * @param value local date + * @param tz time zone, null is treated as UTC * @throws IOException when failed to write value to output stream or reached * end of the stream */ - public static void writeDate32(OutputStream output, LocalDate value) throws IOException { + public static void writeDate32(OutputStream output, LocalDate value, TimeZone tz) throws IOException { writeInt32(output, ClickHouseChecker.between((int) value.toEpochDay(), ClickHouseValues.TYPE_DATE, DATE32_MIN, DATE32_MAX)); } @@ -1241,12 +1338,13 @@ public static void writeDate32(OutputStream output, LocalDate value) throws IOEx * Read {@link java.time.LocalDateTime} from given input stream. * * @param input non-null input stream + * @param tz time zone, null is treated as UTC * @return local datetime * @throws IOException when failed to read value from input stream or reached * end of the stream */ - public static LocalDateTime readDateTime(InputStream input) throws IOException { - return readDateTime(input, 0); + public static LocalDateTime readDateTime(InputStream input, TimeZone tz) throws IOException { + return readDateTime(input, 0, tz); } /** @@ -1254,13 +1352,15 @@ public static LocalDateTime readDateTime(InputStream input) throws IOException { * * @param input non-null input stream * @param scale scale of the datetime, must between 0 and 9 inclusive + * @param tz time zone, null is treated as UTC * @return local datetime * @throws IOException when failed to read value from input stream or reached * end of the stream */ - public static LocalDateTime readDateTime(InputStream input, int scale) throws IOException { - return ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 9) == 0 ? readDateTime32(input) - : readDateTime64(input, scale); + public static LocalDateTime readDateTime(InputStream input, int scale, TimeZone tz) throws IOException { + return ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, + ClickHouseDataType.DateTime64.getMaxScale()) == 0 ? readDateTime32(input, tz) + : readDateTime64(input, scale, tz); } /** @@ -1268,11 +1368,12 @@ public static LocalDateTime readDateTime(InputStream input, int scale) throws IO * * @param output non-null output stream * @param value local datetime + * @param tz time zone, null is treated as UTC * @throws IOException when failed to write value to output stream or reached * end of the stream */ - public static void writeDateTime(OutputStream output, LocalDateTime value) throws IOException { - writeDateTime(output, value, 0); + public static void writeDateTime(OutputStream output, LocalDateTime value, TimeZone tz) throws IOException { + writeDateTime(output, value, 0, tz); } /** @@ -1281,14 +1382,17 @@ public static void writeDateTime(OutputStream output, LocalDateTime value) throw * @param output non-null output stream * @param value local datetime * @param scale scale of the datetime, must between 0 and 9 inclusive + * @param tz time zone, null is treated as UTC * @throws IOException when failed to write value to output stream or reached * end of the stream */ - public static void writeDateTime(OutputStream output, LocalDateTime value, int scale) throws IOException { - if (ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 9) == 0) { - writeDateTime32(output, value); + public static void writeDateTime(OutputStream output, LocalDateTime value, int scale, TimeZone tz) + throws IOException { + if (ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, + ClickHouseDataType.DateTime64.getMaxScale()) == 0) { + writeDateTime32(output, value, tz); } else { - writeDateTime64(output, value, scale); + writeDateTime64(output, value, scale, tz); } } @@ -1296,14 +1400,16 @@ public static void writeDateTime(OutputStream output, LocalDateTime value, int s * Read {@link java.time.LocalDateTime} from given input stream. * * @param input non-null input stream + * @param tz time zone, null is treated as UTC * @return local datetime * @throws IOException when failed to read value from input stream or reached * end of the stream */ - public static LocalDateTime readDateTime32(InputStream input) throws IOException { + public static LocalDateTime readDateTime32(InputStream input, TimeZone tz) throws IOException { long time = readUnsignedInt32(input); - return LocalDateTime.ofEpochSecond(time < 0L ? 0L : time, 0, ZoneOffset.UTC); + return LocalDateTime.ofInstant(Instant.ofEpochSecond(time < 0L ? 0L : time), + tz != null ? tz.toZoneId() : ClickHouseValues.UTC_ZONE); } /** @@ -1311,11 +1417,13 @@ public static LocalDateTime readDateTime32(InputStream input) throws IOException * * @param output non-null output stream * @param value local datetime + * @param tz time zone, null is treated as UTC * @throws IOException when failed to write value to output stream or reached * end of the stream */ - public static void writeDateTime32(OutputStream output, LocalDateTime value) throws IOException { - long time = value.toEpochSecond(ZoneOffset.UTC); + public static void writeDateTime32(OutputStream output, LocalDateTime value, TimeZone tz) throws IOException { + long time = tz == null || tz.equals(ClickHouseValues.UTC_TIMEZONE) ? value.toEpochSecond(ZoneOffset.UTC) + : value.atZone(tz.toZoneId()).toEpochSecond(); writeUnsignedInt32(output, ClickHouseChecker.between(time, ClickHouseValues.TYPE_DATE_TIME, 0L, DATETIME_MAX)); } @@ -1325,12 +1433,13 @@ public static void writeDateTime32(OutputStream output, LocalDateTime value) thr * {@code readDateTime64(input, 3)}. * * @param input non-null input stream + * @param tz time zone, null is treated as UTC * @return local datetime * @throws IOException when failed to read value from input stream or reached * end of the stream */ - public static LocalDateTime readDateTime64(InputStream input) throws IOException { - return readDateTime64(input, 3); + public static LocalDateTime readDateTime64(InputStream input, TimeZone tz) throws IOException { + return readDateTime64(input, 3, tz); } /** @@ -1338,11 +1447,12 @@ public static LocalDateTime readDateTime64(InputStream input) throws IOException * * @param input non-null input stream * @param scale scale of the datetime + * @param tz time zone, null is treated as UTC * @return local datetime * @throws IOException when failed to read value from input stream or reached * end of the stream */ - public static LocalDateTime readDateTime64(InputStream input, int scale) throws IOException { + public static LocalDateTime readDateTime64(InputStream input, int scale, TimeZone tz) throws IOException { long value = readInt64(input); int nanoSeconds = 0; if (ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 9) > 0) { @@ -1364,7 +1474,8 @@ public static LocalDateTime readDateTime64(InputStream input, int scale) throws } } - return LocalDateTime.ofEpochSecond(value, nanoSeconds, ZoneOffset.UTC); + return LocalDateTime.ofInstant(Instant.ofEpochSecond(value, nanoSeconds), + tz != null ? tz.toZoneId() : ClickHouseValues.UTC_ZONE); } /** @@ -1373,11 +1484,12 @@ public static LocalDateTime readDateTime64(InputStream input, int scale) throws * * @param output non-null output stream * @param value local datetime + * @param tz time zone, null is treated as UTC * @throws IOException when failed to write value to output stream or reached * end of the stream */ - public static void writeDateTime64(OutputStream output, LocalDateTime value) throws IOException { - writeDateTime64(output, value, 3); + public static void writeDateTime64(OutputStream output, LocalDateTime value, TimeZone tz) throws IOException { + writeDateTime64(output, value, 3, tz); } /** @@ -1386,12 +1498,16 @@ public static void writeDateTime64(OutputStream output, LocalDateTime value) thr * @param output non-null output stream * @param value local datetime * @param scale scale of the datetime, must between 0 and 9 inclusive + * @param tz time zone, null is treated as UTC * @throws IOException when failed to write value to output stream or reached * end of the stream */ - public static void writeDateTime64(OutputStream output, LocalDateTime value, int scale) throws IOException { - long v = ClickHouseChecker.between(value.toEpochSecond(ZoneOffset.UTC), ClickHouseValues.TYPE_DATE_TIME, - DATETIME64_MIN, DATETIME64_MAX); + public static void writeDateTime64(OutputStream output, LocalDateTime value, int scale, TimeZone tz) + throws IOException { + long v = ClickHouseChecker.between( + tz == null || tz.equals(ClickHouseValues.UTC_TIMEZONE) ? value.toEpochSecond(ZoneOffset.UTC) + : value.atZone(tz.toZoneId()).toEpochSecond(), + ClickHouseValues.TYPE_DATE_TIME, DATETIME64_MIN, DATETIME64_MAX); if (ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 9) > 0) { for (int i = 0; i < scale; i++) { v *= 10; @@ -1480,7 +1596,7 @@ public static void writeFixedString(OutputStream output, String value, int lengt * end of the stream */ public static String readString(InputStream input) throws IOException { - return readString(input, null); + return readString(input, readVarInt(input), null); } /** @@ -1493,7 +1609,34 @@ public static String readString(InputStream input) throws IOException { * end of the stream */ public static String readString(InputStream input, Charset charset) throws IOException { - return new String(readBytes(input, readVarInt(input)), charset == null ? StandardCharsets.UTF_8 : charset); + return readString(input, readVarInt(input), charset); + } + + /** + * Reads fixed string from given input stream. + * + * @param input non-null input stream + * @param length length in byte + * @param charset charset used to convert byte array to string, null means UTF-8 + * @return string value + * @throws IOException when failed to read value from input stream or reached + * end of the stream + */ + public static String readString(InputStream input, int length, Charset charset) throws IOException { + return new String(readBytes(input, length), charset == null ? StandardCharsets.UTF_8 : charset); + } + + /** + * Reads characters from given reader. + * + * @param input non-null reader + * @param length length in character + * @return string value + * @throws IOException when failed to read value from reader or reached end of + * the stream + */ + public static String readString(Reader input, int length) throws IOException { + return new String(readCharacters(input, length)); } /** diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseArrayValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseArrayValue.java index 6973e023f..107f63ed3 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseArrayValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseArrayValue.java @@ -35,7 +35,7 @@ public class ClickHouseArrayValue extends ClickHouseObjectValue { */ @SuppressWarnings("unchecked") public static ClickHouseArrayValue ofEmpty() { - return of((T[]) ClickHouseValues.EMPTY_ARRAY); + return of((T[]) ClickHouseValues.EMPTY_OBJECT_ARRAY); } /** @@ -136,7 +136,7 @@ public boolean isNullOrEmpty() { @Override @SuppressWarnings("unchecked") public ClickHouseArrayValue resetToNullOrEmpty() { - set((T[]) ClickHouseValues.EMPTY_ARRAY); + set((T[]) ClickHouseValues.EMPTY_OBJECT_ARRAY); return this; } @@ -149,7 +149,7 @@ public String toSqlExpression() { StringBuilder builder = new StringBuilder().append('['); for (T v : value) { - builder.append(ClickHouseValues.convertToSqlExpression(v)); + builder.append(ClickHouseValues.convertToSqlExpression(v)).append(','); } if (builder.length() > 1) { builder.setLength(builder.length() - 1); @@ -338,57 +338,81 @@ public ClickHouseArrayValue update(double[] value) { @Override @SuppressWarnings("unchecked") public ClickHouseArrayValue update(BigInteger value) { - set((T[]) new BigInteger[] { value }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + + return set((T[]) new BigInteger[] { value }); } @Override @SuppressWarnings("unchecked") public ClickHouseArrayValue update(BigDecimal value) { - set((T[]) new BigDecimal[] { value }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + + return set((T[]) new BigDecimal[] { value }); } @Override @SuppressWarnings("unchecked") public ClickHouseArrayValue update(Enum value) { - set((T[]) new Enum[] { value }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + + return set((T[]) new Enum[] { value }); } @Override @SuppressWarnings("unchecked") public ClickHouseArrayValue update(Inet4Address value) { - set((T[]) new Inet4Address[] { value }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + + return set((T[]) new Inet4Address[] { value }); } @Override @SuppressWarnings("unchecked") public ClickHouseArrayValue update(Inet6Address value) { - set((T[]) new Inet6Address[] { value }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + + return set((T[]) new Inet6Address[] { value }); } @Override @SuppressWarnings("unchecked") public ClickHouseArrayValue update(LocalDate value) { - set((T[]) new LocalDate[] { value }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + + return set((T[]) new LocalDate[] { value }); } @Override @SuppressWarnings("unchecked") public ClickHouseArrayValue update(LocalTime value) { - set((T[]) new LocalTime[] { value }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + + return set((T[]) new LocalTime[] { value }); } @Override @SuppressWarnings("unchecked") public ClickHouseArrayValue update(LocalDateTime value) { - set((T[]) new LocalDateTime[] { value }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + + return set((T[]) new LocalDateTime[] { value }); } @Override @@ -462,7 +486,7 @@ public ClickHouseArrayValue update(ClickHouseValue value) { } else if (value instanceof ClickHouseArrayValue) { set(((ClickHouseArrayValue) value).getValue()); } else { - set(value.isNullOrEmpty() ? (T[]) ClickHouseValues.EMPTY_ARRAY : (T[]) value.asArray()); + set(value.isNullOrEmpty() ? (T[]) ClickHouseValues.EMPTY_OBJECT_ARRAY : (T[]) value.asArray()); } return this; } @@ -470,7 +494,7 @@ public ClickHouseArrayValue update(ClickHouseValue value) { @Override @SuppressWarnings("unchecked") public ClickHouseArrayValue update(Object[] value) { - if (value == null || value.length == 0) { + if (value == null) { return resetToNullOrEmpty(); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBitmap.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBitmap.java index de0d46659..a6db873cd 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBitmap.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBitmap.java @@ -6,6 +6,7 @@ import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -18,6 +19,21 @@ import com.clickhouse.client.ClickHouseDataType; public abstract class ClickHouseBitmap { + private static final int[] EMPTY_INT_ARRAY = new int[0]; + private static final long[] EMPTY_LONG_ARRAY = new long[0]; + private static final ClickHouseBitmap EMPTY_INT8_BITMAP = wrap(ImmutableRoaringBitmap.bitmapOf(EMPTY_INT_ARRAY), + ClickHouseDataType.Int8); + private static final ClickHouseBitmap EMPTY_UINT8_BITMAP = wrap(ImmutableRoaringBitmap.bitmapOf(EMPTY_INT_ARRAY), + ClickHouseDataType.UInt8); + private static final ClickHouseBitmap EMPTY_INT16_BITMAP = wrap(ImmutableRoaringBitmap.bitmapOf(EMPTY_INT_ARRAY), + ClickHouseDataType.Int16); + private static final ClickHouseBitmap EMPTY_UINT16_BITMAP = wrap(ImmutableRoaringBitmap.bitmapOf(EMPTY_INT_ARRAY), + ClickHouseDataType.UInt16); + private static final ClickHouseBitmap EMPTY_INT32_BITMAP = wrap(ImmutableRoaringBitmap.bitmapOf(EMPTY_INT_ARRAY), + ClickHouseDataType.Int32); + private static final ClickHouseBitmap EMPTY_UINT32_BITMAP = wrap(ImmutableRoaringBitmap.bitmapOf(EMPTY_INT_ARRAY), + ClickHouseDataType.UInt32); + static class ClickHouseRoaringBitmap extends ClickHouseBitmap { private final RoaringBitmap rb; @@ -27,6 +43,11 @@ protected ClickHouseRoaringBitmap(RoaringBitmap bitmap, ClickHouseDataType inner this.rb = Objects.requireNonNull(bitmap); } + @Override + public boolean isEmpty() { + return rb.isEmpty(); + } + @Override public int getCardinality() { return rb.getCardinality(); @@ -57,6 +78,11 @@ protected ClickHouseImmutableRoaringBitmap(ImmutableRoaringBitmap rb, ClickHouse this.rb = Objects.requireNonNull(rb); } + @Override + public boolean isEmpty() { + return rb.isEmpty(); + } + @Override public int getCardinality() { return rb.getCardinality(); @@ -87,6 +113,11 @@ protected ClickHouseMutableRoaringBitmap(MutableRoaringBitmap bitmap, ClickHouse this.rb = Objects.requireNonNull(bitmap); } + @Override + public boolean isEmpty() { + return rb.isEmpty(); + } + @Override public int getCardinality() { return rb.getCardinality(); @@ -117,6 +148,11 @@ protected ClickHouseRoaring64NavigableMap(Roaring64NavigableMap bitmap, ClickHou this.rb = Objects.requireNonNull(bitmap); } + @Override + public boolean isEmpty() { + return rb.isEmpty(); + } + @Override public int getCardinality() { return rb.getIntCardinality(); @@ -178,6 +214,45 @@ public long[] toLongArray() { } } + public static ClickHouseBitmap empty() { + return empty(null); + } + + public static ClickHouseBitmap empty(ClickHouseDataType type) { + if (type == null) { + type = ClickHouseDataType.UInt32; + } + + ClickHouseBitmap v; + switch (type) { + case Int8: + v = ClickHouseBitmap.EMPTY_INT8_BITMAP; + break; + case UInt8: + v = ClickHouseBitmap.EMPTY_UINT8_BITMAP; + break; + case Int16: + v = ClickHouseBitmap.EMPTY_INT16_BITMAP; + break; + case UInt16: + v = ClickHouseBitmap.EMPTY_UINT16_BITMAP; + break; + case Int32: + v = ClickHouseBitmap.EMPTY_INT32_BITMAP; + break; + case UInt32: + v = ClickHouseBitmap.EMPTY_UINT32_BITMAP; + break; + case Int64: + case UInt64: + v = wrap(Roaring64NavigableMap.bitmapOf(EMPTY_LONG_ARRAY), type); + break; + default: + throw new IllegalArgumentException("Only native integer types are supported but we got: " + type.name()); + } + return v; + } + public static ClickHouseBitmap wrap(byte... values) { boolean isUnsigned = true; int len = values.length; @@ -259,6 +334,10 @@ public static ClickHouseBitmap wrap(Object bitmap, ClickHouseDataType innerType) return b; } + public static ClickHouseBitmap deserialize(InputStream in, ClickHouseDataType innerType) throws IOException { + return deserialize(in instanceof DataInputStream ? (DataInputStream) in : new DataInputStream(in), innerType); + } + public static ClickHouseBitmap deserialize(DataInputStream in, ClickHouseDataType innerType) throws IOException { final ClickHouseBitmap rb; @@ -396,27 +475,20 @@ private static ByteBuffer flip(ByteBuffer buffer) { } private static int byteLength(ClickHouseDataType type) { - int byteLen = 0; + int byteLen; switch (Objects.requireNonNull(type)) { - case Int8: - case UInt8: - byteLen = 1; - break; - case Int16: - case UInt16: - byteLen = 2; - break; - case Int32: - case UInt32: - byteLen = 4; - break; - case Int64: - case UInt64: - byteLen = 8; - break; - default: - throw new IllegalArgumentException( - "Only native integer types are supported but we got: " + type.name()); + case Int8: + case UInt8: + case Int16: + case UInt16: + case Int32: + case UInt32: + case Int64: + case UInt64: + byteLen = type.getByteLength(); + break; + default: + throw new IllegalArgumentException("Only native integer types are supported but we got: " + type.name()); } return byteLen; @@ -432,6 +504,8 @@ protected ClickHouseBitmap(Object bitmap, ClickHouseDataType innerType) { this.reference = Objects.requireNonNull(bitmap); } + public abstract boolean isEmpty(); + public abstract int getCardinality(); public long getLongCardinality() { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBitmapValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBitmapValue.java new file mode 100644 index 000000000..e494b26fa --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBitmapValue.java @@ -0,0 +1,350 @@ +package com.clickhouse.client.data; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.UUID; + +import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.client.ClickHouseValues; + +/** + * Wraper class of Bitmap. + */ +public class ClickHouseBitmapValue extends ClickHouseObjectValue { + /** + * Create a new instance representing empty value. + * + * @param valueType value type, must be native integer + * @return new instance representing empty value + */ + public static ClickHouseBitmapValue ofEmpty(ClickHouseDataType valueType) { + return ofEmpty(null, valueType); + } + + /** + * Update given value to empty or create a new instance if {@code ref} is null. + * + * @param ref object to update, could be null + * @param valueType value type, must be native integer + * @return same object as {@code ref} or a new instance if it's null + */ + public static ClickHouseBitmapValue ofEmpty(ClickHouseValue ref, ClickHouseDataType valueType) { + ClickHouseBitmap v = ClickHouseBitmap.empty(valueType); + return ref instanceof ClickHouseBitmapValue ? (ClickHouseBitmapValue) ((ClickHouseBitmapValue) ref).set(v) + : new ClickHouseBitmapValue(v); + } + + /** + * Wrap the given value. + * + * @param value value + * @return object representing the value + */ + public static ClickHouseBitmapValue of(ClickHouseBitmap value) { + return of(null, value); + } + + /** + * Update value of the given object or create a new instance if {@code ref} is + * null. + * + * @param ref object to update, could be null + * @param value value + * @return same object as {@code ref} or a new instance if it's null + */ + public static ClickHouseBitmapValue of(ClickHouseValue ref, ClickHouseBitmap value) { + if (value == null) { + value = ClickHouseBitmap.empty(); + } + + return ref instanceof ClickHouseBitmapValue ? (ClickHouseBitmapValue) ((ClickHouseBitmapValue) ref).set(value) + : new ClickHouseBitmapValue(value); + } + + protected ClickHouseBitmapValue(ClickHouseBitmap value) { + super(value); + } + + @Override + public ClickHouseBitmapValue copy(boolean deep) { + return new ClickHouseBitmapValue(getValue()); + } + + @Override + public boolean isNullOrEmpty() { + return getValue().isEmpty(); + } + + @Override + public byte asByte() { + if (isNullOrEmpty()) { + return (byte) 0; + } + + ClickHouseBitmap v = getValue(); + if (v.getCardinality() != 1) { + throw new IllegalArgumentException( + ClickHouseUtils.format("Expect only one element but we got %d", v.getLongCardinality())); + } + return v.innerType.getByteLength() > 4 ? (byte) v.toLongArray()[0] : (byte) v.toIntArray()[0]; + } + + @Override + public short asShort() { + if (isNullOrEmpty()) { + return (short) 0; + } + + ClickHouseBitmap v = getValue(); + if (v.getCardinality() != 1) { + throw new IllegalArgumentException( + ClickHouseUtils.format("Expect only one element but we got %d", v.getLongCardinality())); + } + return v.innerType.getByteLength() > 4 ? (short) v.toLongArray()[0] : (short) v.toIntArray()[0]; + } + + @Override + public int asInteger() { + if (isNullOrEmpty()) { + return 0; + } + + ClickHouseBitmap v = getValue(); + if (v.getCardinality() != 1) { + throw new IllegalArgumentException( + ClickHouseUtils.format("Expect only one element but we got %d", v.getLongCardinality())); + } + return v.innerType.getByteLength() > 4 ? (int) v.toLongArray()[0] : v.toIntArray()[0]; + } + + @Override + public long asLong() { + if (isNullOrEmpty()) { + return 0; + } + + ClickHouseBitmap v = getValue(); + if (v.getCardinality() != 1) { + throw new IllegalArgumentException( + ClickHouseUtils.format("Expect only one element but we got %d", v.getLongCardinality())); + } + return v.innerType.getByteLength() > 4 ? v.toLongArray()[0] : v.toIntArray()[0]; + } + + @Override + public BigInteger asBigInteger() { + if (isNullOrEmpty()) { + return null; + } + + return BigInteger.valueOf(asLong()); + } + + @Override + public float asFloat() { + return isNullOrEmpty() ? 0F : (float) asInteger(); + } + + @Override + public double asDouble() { + return isNullOrEmpty() ? 0D : (double) asLong(); + } + + @Override + public BigDecimal asBigDecimal(int scale) { + return isNullOrEmpty() ? null : new BigDecimal(asBigInteger()).setScale(scale); + } + + @Override + public Object asObject() { + return getValue(); + } + + public long getCardinality() { + return isNullOrEmpty() ? 0L : getValue().getLongCardinality(); + } + + @Override + public String toSqlExpression() { + return isNullOrEmpty() ? ClickHouseValues.NULL_EXPR : String.valueOf(getValue()); + } + + @Override + public ClickHouseBitmapValue update(boolean value) { + set(ClickHouseBitmap.wrap(value ? (byte) 1 : (byte) 0)); + return this; + } + + @Override + public ClickHouseBitmapValue update(char value) { + set(ClickHouseBitmap.wrap((short) value)); + return this; + } + + @Override + public ClickHouseBitmapValue update(byte value) { + set(ClickHouseBitmap.wrap(value)); + return this; + } + + @Override + public ClickHouseBitmapValue update(short value) { + set(ClickHouseBitmap.wrap(value)); + return this; + } + + @Override + public ClickHouseBitmapValue update(int value) { + set(ClickHouseBitmap.wrap(value)); + return this; + } + + @Override + public ClickHouseBitmapValue update(long value) { + set(ClickHouseBitmap.wrap(value)); + return this; + } + + @Override + public ClickHouseBitmapValue update(float value) { + set(ClickHouseBitmap.wrap((int) value)); + return this; + } + + @Override + public ClickHouseBitmapValue update(double value) { + set(ClickHouseBitmap.wrap((long) value)); + return this; + } + + @Override + public ClickHouseBitmapValue update(BigInteger value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + set(ClickHouseBitmap.wrap(value.longValue())); + } + return this; + } + + @Override + public ClickHouseBitmapValue update(BigDecimal value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + set(ClickHouseBitmap.wrap(value.longValue())); + } + return this; + } + + @Override + public ClickHouseBitmapValue update(Enum value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + set(ClickHouseBitmap.wrap(value.ordinal())); + } + return this; + } + + @Override + public ClickHouseBitmapValue update(Inet4Address value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + set(ClickHouseBitmap.wrap(ClickHouseValues.convertToBigInteger(value).longValue())); + } + return this; + } + + @Override + public ClickHouseBitmapValue update(Inet6Address value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + set(ClickHouseBitmap.wrap(ClickHouseValues.convertToBigInteger(value).longValue())); + } + return this; + } + + @Override + public ClickHouseBitmapValue update(LocalDate value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + set(ClickHouseBitmap.wrap(value.toEpochDay())); + } + return this; + } + + @Override + public ClickHouseBitmapValue update(LocalTime value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + set(ClickHouseBitmap.wrap(value.toSecondOfDay())); + } + return this; + } + + @Override + public ClickHouseBitmapValue update(LocalDateTime value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + set(ClickHouseBitmap.wrap(value.toEpochSecond(ZoneOffset.UTC))); + } + return this; + } + + @Override + public ClickHouseBitmapValue update(String value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + set(ClickHouseBitmap.wrap(Long.parseLong(value))); + } + return this; + } + + @Override + public ClickHouseBitmapValue update(UUID value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + set(ClickHouseBitmap.wrap(ClickHouseValues.convertToBigInteger(value).longValue())); + } + return this; + } + + @Override + public ClickHouseBitmapValue update(ClickHouseValue value) { + if (value == null) { + resetToNullOrEmpty(); + } else if (value instanceof ClickHouseBitmapValue) { + set(((ClickHouseBitmapValue) value).getValue()); + } else { + set(ClickHouseBitmap.wrap(value.asInteger())); + } + return this; + } + + @Override + public ClickHouseBitmapValue update(Object value) { + if (value instanceof ClickHouseBitmap) { + set((ClickHouseBitmap) value); + return this; + } + + super.update(value); + return this; + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateTimeValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateTimeValue.java index 6909129c7..c7ae6a7e5 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateTimeValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateTimeValue.java @@ -10,6 +10,7 @@ import java.time.LocalTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; + import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseValue; import com.clickhouse.client.ClickHouseValues; @@ -18,7 +19,7 @@ * Wraper class of LocalDateTime. */ public class ClickHouseDateTimeValue extends ClickHouseObjectValue { - private static final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + static final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); /** * Create a new instance representing null getValue(). @@ -166,12 +167,7 @@ public BigDecimal asBigDecimal(int scale) { @Override public LocalDate asDate() { - return isNullOrEmpty() ? null : getValue().toLocalDate(); - } - - @Override - public LocalTime asTime() { - return isNullOrEmpty() ? null : getValue().toLocalTime(); + return isNullOrEmpty() ? null : asDateTime(0).toLocalDate(); } @Override @@ -205,6 +201,7 @@ public String toSqlExpression() { if (isNullOrEmpty()) { return ClickHouseValues.NULL_EXPR; } + return new StringBuilder().append('\'') .append(getValue().format(scale > 0 ? ClickHouseValues.DATETIME_FORMATTER : dateTimeFormatter)) .append('\'').toString(); diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateValue.java index 3cc9e6551..3eb48abca 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateValue.java @@ -7,6 +7,7 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; + import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseValue; import com.clickhouse.client.ClickHouseValues; @@ -45,6 +46,12 @@ public static ClickHouseDateValue of(LocalDate value) { return of(null, value); } + /** + * Wrap the given value. + * + * @param epochDay epoch day + * @return object representing the value + */ public static ClickHouseDateValue of(long epochDay) { return of(null, LocalDate.ofEpochDay(epochDay)); } @@ -117,7 +124,7 @@ public LocalDate asDate() { } @Override - public LocalTime asTime() { + public final LocalTime asTime(int scale) { return isNullOrEmpty() ? null : LocalTime.ofSecondOfDay(0L); } @@ -136,7 +143,7 @@ public String asString(int length, Charset charset) { return null; } - String str = getValue().format(ClickHouseValues.DATE_FORMATTER); + String str = asDate().format(ClickHouseValues.DATE_FORMATTER); if (length > 0) { ClickHouseChecker.notWithDifferentLength(str.getBytes(charset == null ? StandardCharsets.UTF_8 : charset), length); @@ -150,7 +157,8 @@ public String toSqlExpression() { if (isNullOrEmpty()) { return ClickHouseValues.NULL_EXPR; } - return new StringBuilder().append('\'').append(getValue().format(ClickHouseValues.DATE_FORMATTER)).append('\'').toString(); + return new StringBuilder().append('\'').append(asDate().format(ClickHouseValues.DATE_FORMATTER)).append('\'') + .toString(); } @Override diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDoubleValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDoubleValue.java index 98b2f4a9d..fb9703408 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDoubleValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDoubleValue.java @@ -130,7 +130,7 @@ public BigDecimal asBigDecimal(int scale) { } BigDecimal dec = BigDecimal.valueOf(value); - if (value == 0F || isInfinity() || isNaN()) { + if (value == 0D || isInfinity() || isNaN()) { dec = dec.setScale(scale); } else { int diff = scale - dec.scale(); diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseEmptyValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseEmptyValue.java new file mode 100644 index 000000000..36a847aa7 --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseEmptyValue.java @@ -0,0 +1,139 @@ +package com.clickhouse.client.data; + +import java.math.BigDecimal; +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseValue; + +/** + * Wrapper class of Nothing. + */ +public final class ClickHouseEmptyValue implements ClickHouseValue { + /** + * Singleton. + */ + public static final ClickHouseEmptyValue INSTANCE = new ClickHouseEmptyValue(); + + @Override + public BigDecimal asBigDecimal(int scale) { + return null; + } + + @Override + public BigInteger asBigInteger() { + return null; + } + + @Override + public byte asByte() { + throw new IllegalStateException("Empty value cannot be converted to byte"); + } + + @Override + public double asDouble() { + throw new IllegalStateException("Empty value cannot be converted to double"); + } + + @Override + public float asFloat() { + throw new IllegalStateException("Empty value cannot be converted to float"); + } + + @Override + public int asInteger() { + throw new IllegalStateException("Empty value cannot be converted to int"); + } + + @Override + public long asLong() { + throw new IllegalStateException("Empty value cannot be converted to long"); + } + + @Override + public Object asObject() { + return null; + } + + @Override + public short asShort() { + throw new IllegalStateException("Empty value cannot be converted to short"); + } + + @Override + public ClickHouseValue copy(boolean deep) { + return INSTANCE; + } + + @Override + public boolean isNullOrEmpty() { + return true; + } + + @Override + public ClickHouseValue resetToNullOrEmpty() { + return INSTANCE; + } + + @Override + public String toSqlExpression() { + return toString(); + } + + @Override + public ClickHouseValue update(byte value) { + return INSTANCE; + } + + @Override + public ClickHouseValue update(short value) { + return INSTANCE; + } + + @Override + public ClickHouseValue update(int value) { + return INSTANCE; + } + + @Override + public ClickHouseValue update(long value) { + return INSTANCE; + } + + @Override + public ClickHouseValue update(float value) { + return INSTANCE; + } + + @Override + public ClickHouseValue update(double value) { + return INSTANCE; + } + + @Override + public ClickHouseValue update(BigInteger value) { + return INSTANCE; + } + + @Override + public ClickHouseValue update(BigDecimal value) { + return INSTANCE; + } + + @Override + public ClickHouseValue update(String value) { + return INSTANCE; + } + + @Override + public ClickHouseValue update(ClickHouseValue value) { + return INSTANCE; + } + + @Override + public String toString() { + return ""; + } + + private ClickHouseEmptyValue() { + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseExternalTable.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseExternalTable.java index 7e0457978..e7af11102 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseExternalTable.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseExternalTable.java @@ -7,6 +7,10 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; + import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseFormat; @@ -14,7 +18,7 @@ public class ClickHouseExternalTable { public static class Builder { private String name; - private InputStream content; + private CompletableFuture content; private ClickHouseFormat format; private List columns; @@ -28,7 +32,12 @@ public Builder name(String name) { } public Builder content(InputStream content) { - this.content = content; + this.content = CompletableFuture.completedFuture(ClickHouseChecker.nonNull(content, "content")); + return this; + } + + public Builder content(CompletableFuture content) { + this.content = ClickHouseChecker.nonNull(content, "Content"); return this; } @@ -89,13 +98,13 @@ public static Builder builder() { } private final String name; - private final InputStream content; + private final CompletableFuture content; private final ClickHouseFormat format; private final List columns; private final String structure; - protected ClickHouseExternalTable(String name, InputStream content, ClickHouseFormat format, + protected ClickHouseExternalTable(String name, CompletableFuture content, ClickHouseFormat format, Collection columns) { this.name = name == null ? "" : name.trim(); this.content = ClickHouseChecker.nonNull(content, "content"); @@ -126,7 +135,14 @@ public String getName() { } public InputStream getContent() { - return content; + try { + return content.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new CompletionException(e); + } catch (ExecutionException e) { + throw new CompletionException(e.getCause()); + } } public ClickHouseFormat getFormat() { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseLZ4InputStream.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseLZ4InputStream.java index 8435ea0be..6bb62f356 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseLZ4InputStream.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseLZ4InputStream.java @@ -1,8 +1,11 @@ package com.clickhouse.client.data; -import java.io.DataInputStream; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; + +import com.clickhouse.client.ClickHouseChecker; + import net.jpountz.lz4.LZ4Factory; import net.jpountz.lz4.LZ4FastDecompressor; @@ -15,23 +18,48 @@ public class ClickHouseLZ4InputStream extends InputStream { static final int MAGIC = 0x82; private final InputStream stream; - private final DataInputStream dataWrapper; private byte[] currentBlock; private int pointer; + private void readFully(byte b[], int off, int len) throws IOException { + if (len < 0) { + throw new IndexOutOfBoundsException(); + } + int n = 0; + while (n < len) { + int count = stream.read(b, off + n, len - n); + if (count < 0) { + throw new EOFException(); + } + n += count; + } + } + + private int readUnsignedByte() throws IOException { + int ch = stream.read(); + if (ch < 0) { + throw new EOFException(); + } + return ch; + } + public ClickHouseLZ4InputStream(InputStream stream) { - this.stream = stream; - dataWrapper = new DataInputStream(stream); + this.stream = ClickHouseChecker.nonNull(stream, "InputStream"); + } + + @Override + public int available() throws IOException { + int estimated = stream.available(); + if (estimated == 0 && currentBlock != null) { + estimated = currentBlock.length - pointer; + } + return estimated; } @Override public int read() throws IOException { - if (!checkNext()) - return -1; - byte b = currentBlock[pointer]; - pointer += 1; - return b & 0xFF; + return checkNext() ? 0xFF & currentBlock[pointer++] : -1; } @Override @@ -56,7 +84,7 @@ public int read(byte[] b, int off, int len) throws IOException { pointer += toCopy; copied += toCopy; if (!checkNext()) { // finished - return copied; + break; } } return copied; @@ -72,14 +100,14 @@ private boolean checkNext() throws IOException { currentBlock = readNextBlock(); pointer = 0; } - return currentBlock != null; + return currentBlock != null && pointer < currentBlock.length; } private int readInt() throws IOException { - byte b1 = (byte) dataWrapper.readUnsignedByte(); - byte b2 = (byte) dataWrapper.readUnsignedByte(); - byte b3 = (byte) dataWrapper.readUnsignedByte(); - byte b4 = (byte) dataWrapper.readUnsignedByte(); + byte b1 = (byte) readUnsignedByte(); + byte b2 = (byte) readUnsignedByte(); + byte b3 = (byte) readUnsignedByte(); + byte b4 = (byte) readUnsignedByte(); return b4 << 24 | (b3 & 0xFF) << 16 | (b2 & 0xFF) << 8 | (b1 & 0xFF); } @@ -93,11 +121,11 @@ private byte[] readNextBlock() throws IOException { byte[] checksum = new byte[16]; checksum[0] = (byte) read; // checksum - 16 bytes. - dataWrapper.readFully(checksum, 1, 15); + readFully(checksum, 1, 15); ClickHouseBlockChecksum expected = ClickHouseBlockChecksum.fromBytes(checksum); // header: // 1 byte - 0x82 (shows this is LZ4) - int magic = dataWrapper.readUnsignedByte(); + int magic = readUnsignedByte(); if (magic != MAGIC) throw new IOException("Magic is not correct: " + magic); // 4 bytes - size of the compressed data including 9 bytes of the header @@ -107,7 +135,7 @@ private byte[] readNextBlock() throws IOException { int compressedSize = compressedSizeWithHeader - 9; // header byte[] block = new byte[compressedSize]; // compressed data: compressed_size - 9 байт. - dataWrapper.readFully(block); + readFully(block, 0, block.length); ClickHouseBlockChecksum real = ClickHouseBlockChecksum.calculateForBlock((byte) magic, compressedSizeWithHeader, uncompressedSize, block, compressedSize); diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseMapValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseMapValue.java index 7dad93f5f..c7af8a0e6 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseMapValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseMapValue.java @@ -10,6 +10,7 @@ import java.time.LocalTime; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.Locale; import java.util.Map; import java.util.UUID; import java.util.Map.Entry; @@ -118,10 +119,28 @@ public ClickHouseMapValue copy(boolean deep) { return new ClickHouseMapValue(newValue, keyType, valueType); } + @Override + @SuppressWarnings("unchecked") + public Map asMap() { + return (Map) getValue(); + } + + @Override + @SuppressWarnings("unchecked") + public Map asMap(Class keyClass, Class valueClass) { + if (!keyType.isAssignableFrom(keyClass) || !valueType.isAssignableFrom(valueClass)) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Incompatible types, expected (%s:%s) but got (%s:%s)", + keyType.getName(), valueType.getName(), keyClass, valueClass)); + } + + return (Map) getValue(); + } + @Override public String asString(int length, Charset charset) { Map value = getValue(); - if (value == null || value.size() == 0) { + if (value == null || value.isEmpty()) { return "{}"; } StringBuilder builder = new StringBuilder().append('{'); @@ -148,7 +167,7 @@ public ClickHouseMapValue resetToNullOrEmpty() { @Override public String toSqlExpression() { Map value = getValue(); - if (value == null || value.size() == 0) { + if (value == null || value.isEmpty()) { return "{}"; } @@ -207,20 +226,29 @@ public ClickHouseMapValue update(double value) { @Override public ClickHouseMapValue update(BigInteger value) { - set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + + return set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); } @Override public ClickHouseMapValue update(BigDecimal value) { - set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + + return set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); } @Override public ClickHouseMapValue update(String value) { - set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + + return set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); } @Override @@ -251,54 +279,67 @@ public ClickHouseMapValue update(Enum value) { } else { throw newUnsupportedException(value.getClass().getName(), valueType.getName()); } - set(Collections.singletonMap(getDefaultKey(), v)); - return this; + return set(Collections.singletonMap(getDefaultKey(), v)); } @Override public ClickHouseMapValue update(Inet4Address value) { - set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); } @Override public ClickHouseMapValue update(Inet6Address value) { - set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); } @Override public ClickHouseMapValue update(LocalDate value) { - set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); } @Override public ClickHouseMapValue update(LocalTime value) { - set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); } @Override public ClickHouseMapValue update(LocalDateTime value) { - set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); } @Override public ClickHouseMapValue update(Map value) { - set(value == null ? Collections.emptyMap() : value); - return this; + return set(value == null ? Collections.emptyMap() : value); } @Override public ClickHouseMapValue update(UUID value) { - set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(Collections.singletonMap(getDefaultKey(), valueType.cast(value))); } @Override public ClickHouseValue updateUnknown(Object value) { + if (value == null) { + return resetToNullOrEmpty(); + } throw new IllegalArgumentException("Unknown value: " + value); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseNestedValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseNestedValue.java index 83105ebc7..4301273bf 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseNestedValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseNestedValue.java @@ -380,56 +380,71 @@ public ClickHouseNestedValue update(double[] value) { v[index++] = d; } } - set(new Object[][] { v }); - return this; + return set(new Object[][] { v }); } @Override public ClickHouseNestedValue update(BigInteger value) { - set(new Object[][] { new BigInteger[] { value } }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(new Object[][] { new BigInteger[] { value } }); } @Override public ClickHouseNestedValue update(BigDecimal value) { - set(new Object[][] { new BigDecimal[] { value } }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(new Object[][] { new BigDecimal[] { value } }); } @Override public ClickHouseNestedValue update(Enum value) { - set(new Object[][] { new Enum[] { value } }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(new Object[][] { new Enum[] { value } }); } @Override public ClickHouseNestedValue update(Inet4Address value) { - set(new Object[][] { new Inet4Address[] { value } }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(new Object[][] { new Inet4Address[] { value } }); } @Override public ClickHouseNestedValue update(Inet6Address value) { - set(new Object[][] { new Inet6Address[] { value } }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(new Object[][] { new Inet6Address[] { value } }); } @Override public ClickHouseNestedValue update(LocalDate value) { - set(new Object[][] { new LocalDate[] { value } }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(new Object[][] { new LocalDate[] { value } }); } @Override public ClickHouseNestedValue update(LocalTime value) { - set(new Object[][] { new LocalTime[] { value } }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(new Object[][] { new LocalTime[] { value } }); } @Override public ClickHouseNestedValue update(LocalDateTime value) { - set(new Object[][] { new LocalDateTime[] { value } }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(new Object[][] { new LocalDateTime[] { value } }); } @Override @@ -444,8 +459,7 @@ public ClickHouseNestedValue update(Collection value) { for (Object o : value) { v[index++] = o; } - set(new Object[][] { v }); - return this; + return set(new Object[][] { v }); } @Override @@ -458,8 +472,7 @@ public ClickHouseNestedValue update(Enumeration value) { while (value.hasMoreElements()) { v.add(value.nextElement()); } - set(new Object[][] { v.toArray(new Object[v.size()]) }); - return this; + return set(new Object[][] { v.toArray(new Object[v.size()]) }); } @Override @@ -474,20 +487,23 @@ public ClickHouseNestedValue update(Map value) { for (Object o : value.values()) { v[index++] = o; } - set(new Object[][] { v }); - return this; + return set(new Object[][] { v }); } @Override public ClickHouseNestedValue update(String value) { - set(new Object[][] { new String[] { value } }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(new Object[][] { new String[] { value } }); } @Override public ClickHouseNestedValue update(UUID value) { - set(new Object[][] { new UUID[] { value } }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(new Object[][] { new UUID[] { value } }); } @Override @@ -516,8 +532,10 @@ public ClickHouseNestedValue update(Object[] value) { @Override public ClickHouseValue updateUnknown(Object value) { - set(new Object[][] { new Object[] { value } }); - return this; + if (value == null) { + return resetToNullOrEmpty(); + } + return set(new Object[][] { new Object[] { value } }); } @Override diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseOffsetDateTimeValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseOffsetDateTimeValue.java new file mode 100644 index 000000000..dbecdfe0e --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseOffsetDateTimeValue.java @@ -0,0 +1,355 @@ +package com.clickhouse.client.data; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.TimeZone; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.client.ClickHouseValues; + +/** + * Wraper class of OffsetDateTime. + */ +public class ClickHouseOffsetDateTimeValue extends ClickHouseObjectValue { + /** + * Create a new instance representing null getValue(). + * + * @param scale scale + * @param tz time zone, null is treated as {@code UTC} + * @return new instance representing null value + */ + public static ClickHouseOffsetDateTimeValue ofNull(int scale, TimeZone tz) { + return ofNull(null, scale, tz); + } + + /** + * Update given value to null or create a new instance if {@code ref} is null. + * + * @param ref object to update, could be null + * @param scale scale, only used when {@code ref} is null + * @param tz time zone, null is treated as {@code UTC} + * @return same object as {@code ref} or a new instance if it's null + */ + public static ClickHouseOffsetDateTimeValue ofNull(ClickHouseValue ref, int scale, TimeZone tz) { + return ref instanceof ClickHouseOffsetDateTimeValue + ? (ClickHouseOffsetDateTimeValue) ((ClickHouseOffsetDateTimeValue) ref).set(null) + : new ClickHouseOffsetDateTimeValue(null, scale, tz); + } + + /** + * Wrap the given getValue(). + * + * @param value value + * @param scale scale + * @param tz time zone, null is treated as {@code UTC} + * @return object representing the value + */ + public static ClickHouseOffsetDateTimeValue of(LocalDateTime value, int scale, TimeZone tz) { + return of(null, value, scale, tz); + } + + /** + * Update value of the given object or create a new instance if {@code ref} is + * null. + * + * @param ref object to update, could be null + * @param value value + * @param scale scale, only used when {@code ref} is null + * @param tz time zone, null is treated as {@code UTC} + * @return same object as {@code ref} or a new instance if it's null + */ + public static ClickHouseOffsetDateTimeValue of(ClickHouseValue ref, LocalDateTime value, int scale, TimeZone tz) { + OffsetDateTime v = null; + if (value != null) { + v = value.atZone( + tz == null || tz.equals(ClickHouseValues.UTC_TIMEZONE) ? ClickHouseValues.UTC_ZONE : tz.toZoneId()) + .toOffsetDateTime(); + } + return ref instanceof ClickHouseOffsetDateTimeValue + ? (ClickHouseOffsetDateTimeValue) ((ClickHouseOffsetDateTimeValue) ref).set(v) + : new ClickHouseOffsetDateTimeValue(v, scale, tz); + } + + private final int scale; + private final TimeZone tz; + + protected ClickHouseOffsetDateTimeValue(OffsetDateTime value, int scale, TimeZone tz) { + super(value); + this.scale = ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 9); + this.tz = tz == null || tz.equals(ClickHouseValues.UTC_TIMEZONE) ? ClickHouseValues.UTC_TIMEZONE : tz; + } + + public int getScale() { + return scale; + } + + @Override + public ClickHouseOffsetDateTimeValue copy(boolean deep) { + return new ClickHouseOffsetDateTimeValue(getValue(), scale, tz); + } + + @Override + public byte asByte() { + return isNullOrEmpty() ? (byte) 0 : (byte) getValue().toEpochSecond(); + } + + @Override + public short asShort() { + return isNullOrEmpty() ? (short) 0 : (short) getValue().toEpochSecond(); + } + + @Override + public int asInteger() { + return isNullOrEmpty() ? 0 : (int) getValue().toEpochSecond(); + } + + @Override + public long asLong() { + return isNullOrEmpty() ? 0L : getValue().toEpochSecond(); + } + + @Override + public float asFloat() { + return isNullOrEmpty() ? 0F + : getValue().toEpochSecond() + getValue().getNano() / ClickHouseValues.NANOS.floatValue(); + } + + @Override + public double asDouble() { + return isNullOrEmpty() ? 0D + : getValue().toEpochSecond() + getValue().getNano() / ClickHouseValues.NANOS.doubleValue(); + } + + @Override + public BigInteger asBigInteger() { + return isNullOrEmpty() ? null : BigInteger.valueOf(getValue().toEpochSecond()); + } + + @Override + public BigDecimal asBigDecimal(int scale) { + OffsetDateTime value = getValue(); + BigDecimal v = null; + if (value != null) { + int nanoSeconds = value.getNano(); + v = new BigDecimal(BigInteger.valueOf(value.toEpochSecond()), scale); + if (scale != 0 && nanoSeconds != 0) { + v = v.add(BigDecimal.valueOf(nanoSeconds).divide(ClickHouseValues.NANOS).setScale(scale, + RoundingMode.HALF_UP)); + } + } + return v; + } + + @Override + public LocalDate asDate() { + return isNullOrEmpty() ? null : asOffsetDateTime(0).toLocalDate(); + } + + @Override + public LocalTime asTime(int scale) { + return isNullOrEmpty() ? null : asOffsetDateTime(scale).toLocalTime(); + } + + @Override + public LocalDateTime asDateTime(int scale) { + if (isNullOrEmpty()) { + return null; + } + + return getValue().toLocalDateTime(); + } + + @Override + public OffsetDateTime asOffsetDateTime(int scale) { + return getValue(); + } + + @Override + public ZonedDateTime asZonedDateTime(int scale) { + if (isNullOrEmpty()) { + return null; + } + + return getValue().toZonedDateTime(); + } + + @Override + public Object asObject() { + return getValue(); + } + + @Override + public String asString(int length, Charset charset) { + if (isNullOrEmpty()) { + return null; + } + + // different formatter for each scale? + String str = asDateTime(scale) + .format(scale > 0 ? ClickHouseValues.DATETIME_FORMATTER : ClickHouseDateTimeValue.dateTimeFormatter); + if (length > 0) { + ClickHouseChecker.notWithDifferentLength( + str.getBytes(charset == null ? StandardCharsets.US_ASCII : charset), length); + } + + return str; + } + + @Override + public String toSqlExpression() { + if (isNullOrEmpty()) { + return ClickHouseValues.NULL_EXPR; + } + + return new StringBuilder().append('\'') + .append(asDateTime(scale).format( + scale > 0 ? ClickHouseValues.DATETIME_FORMATTER : ClickHouseDateTimeValue.dateTimeFormatter)) + .append('\'').toString(); + } + + @Override + public ClickHouseOffsetDateTimeValue update(byte value) { + return update(BigInteger.valueOf(value)); + } + + @Override + public ClickHouseOffsetDateTimeValue update(short value) { + return update(BigInteger.valueOf(value)); + } + + @Override + public ClickHouseOffsetDateTimeValue update(int value) { + return update(BigInteger.valueOf(value)); + } + + @Override + public ClickHouseOffsetDateTimeValue update(long value) { + return update(BigInteger.valueOf(value)); + } + + @Override + public ClickHouseOffsetDateTimeValue update(float value) { + return update(BigDecimal.valueOf(value)); + } + + @Override + public ClickHouseOffsetDateTimeValue update(double value) { + return update(BigDecimal.valueOf(value)); + } + + @Override + public ClickHouseOffsetDateTimeValue update(BigInteger value) { + if (value == null) { + resetToNullOrEmpty(); + } else if (scale == 0) { + set(ClickHouseValues.convertToDateTime(new BigDecimal(value, 0), tz).toOffsetDateTime()); + } else { + set(ClickHouseValues.convertToDateTime(new BigDecimal(value, scale), tz).toOffsetDateTime()); + } + return this; + } + + @Override + public ClickHouseOffsetDateTimeValue update(BigDecimal value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + if (value.scale() != scale) { + value = value.setScale(scale, RoundingMode.HALF_UP); + } + set(ClickHouseValues.convertToDateTime(value, tz).toOffsetDateTime()); + } + return this; + } + + @Override + public ClickHouseOffsetDateTimeValue update(Enum value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + update(BigInteger.valueOf(value.ordinal())); + } + return this; + } + + @Override + public ClickHouseOffsetDateTimeValue update(LocalDate value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + LocalDateTime dateTime = LocalDateTime.of(value, LocalTime.MIN); + set(tz != null ? dateTime.atZone(tz.toZoneId()).toOffsetDateTime() : dateTime.atOffset(ZoneOffset.UTC)); + } + return this; + } + + @Override + public ClickHouseOffsetDateTimeValue update(LocalTime value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + LocalDateTime dateTime = LocalDateTime.of(LocalDate.now(), value); + set(tz != null ? dateTime.atZone(tz.toZoneId()).toOffsetDateTime() : dateTime.atOffset(ZoneOffset.UTC)); + } + return this; + } + + @Override + public ClickHouseOffsetDateTimeValue update(LocalDateTime value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + set(tz != null && !tz.equals(ClickHouseValues.UTC_TIMEZONE) ? value.atZone(tz.toZoneId()).toOffsetDateTime() + : value.atOffset(ZoneOffset.UTC)); + } + return this; + } + + @Override + public ClickHouseOffsetDateTimeValue update(OffsetDateTime value) { + set(value); + return this; + } + + @Override + public ClickHouseOffsetDateTimeValue update(String value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + set(OffsetDateTime.parse(value, ClickHouseValues.DATETIME_FORMATTER)); + } + return this; + } + + @Override + public ClickHouseOffsetDateTimeValue update(ClickHouseValue value) { + if (value == null) { + resetToNullOrEmpty(); + } else { + set(value.asOffsetDateTime(scale)); + } + return this; + } + + @Override + public ClickHouseOffsetDateTimeValue update(Object value) { + if (value instanceof OffsetDateTime) { + set((OffsetDateTime) value); + } else if (value instanceof String) { + update((String) value); + } else { + super.update(value); + } + return this; + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHousePipedStream.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHousePipedStream.java index 366bb5d12..a2481872a 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHousePipedStream.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHousePipedStream.java @@ -46,7 +46,6 @@ private void ensureOpen() throws IOException { } } - @SuppressWarnings("squid:S2142") private int updateBuffer() throws IOException { try { if (timeout > 0) { @@ -60,6 +59,7 @@ private int updateBuffer() throws IOException { return buffer.remaining(); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new IOException("Thread was interrupted when getting next buffer from queue", e); } } @@ -186,7 +186,6 @@ private void ensureOpen() throws IOException { } } - @SuppressWarnings("squid:S2142") private void updateBuffer() throws IOException { if (buffer.position() > 0) { if (buffer.hasRemaining()) { @@ -203,6 +202,7 @@ private void updateBuffer() throws IOException { queue.put(buffer); } } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new IOException("Thread was interrupted when putting buffer into queue", e); } @@ -215,7 +215,6 @@ public InputStream getInput() { } @Override - @SuppressWarnings("squid:S2142") public void close() throws IOException { if (this.closed) { return; @@ -233,6 +232,7 @@ public void close() throws IOException { queue.put(buffer); } } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new IOException("Thread was interrupted when putting EMPTY buffer into queue", e); } this.closed = true; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRecordTransformer.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRecordTransformer.java new file mode 100644 index 000000000..88d258ce8 --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRecordTransformer.java @@ -0,0 +1,14 @@ +package com.clickhouse.client.data; + +import com.clickhouse.client.ClickHouseRecord; + +@FunctionalInterface +public interface ClickHouseRecordTransformer { + /** + * Updates values in the given record. + * + * @param rowIndex zero-based index of row + * @param r record to update + */ + void update(int rowIndex, ClickHouseRecord r); +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessor.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessor.java index 70da0895d..23bfdff71 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessor.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessor.java @@ -4,7 +4,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.lang.reflect.Array; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; import java.util.EnumMap; @@ -12,7 +12,11 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Map.Entry; import java.util.function.Supplier; + +import com.clickhouse.client.ClickHouseAggregateFunction; import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseConfig; @@ -34,6 +38,124 @@ public class ClickHouseRowBinaryProcessor extends ClickHouseDataProcessor { public static class MappedFunctions { private static final MappedFunctions instance = new MappedFunctions(); + private void writeArray(ClickHouseValue value, ClickHouseConfig config, ClickHouseColumn column, + OutputStream output) throws IOException { + ClickHouseColumn nestedColumn = column.getNestedColumns().get(0); + ClickHouseColumn baseColumn = column.getArrayBaseColumn(); + int level = column.getArrayNestedLevel(); + Class javaClass = baseColumn.getDataType().getPrimitiveClass(); + if (level > 1 || !javaClass.isPrimitive()) { + Object[] array = value.asArray(); + ClickHouseValue v = ClickHouseValues.newValue(nestedColumn); + int length = array.length; + BinaryStreamUtils.writeVarInt(output, length); + for (int i = 0; i < length; i++) { + serialize(v.update(array[i]), config, nestedColumn, output); + } + } else { + ClickHouseValue v = ClickHouseValues.newValue(baseColumn); + if (byte.class == javaClass) { + byte[] array = (byte[]) value.asObject(); + int length = array.length; + BinaryStreamUtils.writeVarInt(output, length); + for (int i = 0; i < length; i++) { + serialize(v.update(array[i]), config, baseColumn, output); + } + } else if (short.class == javaClass) { + short[] array = (short[]) value.asObject(); + int length = array.length; + BinaryStreamUtils.writeVarInt(output, length); + for (int i = 0; i < length; i++) { + serialize(v.update(array[i]), config, baseColumn, output); + } + } else if (int.class == javaClass) { + int[] array = (int[]) value.asObject(); + int length = array.length; + BinaryStreamUtils.writeVarInt(output, length); + for (int i = 0; i < length; i++) { + serialize(v.update(array[i]), config, baseColumn, output); + } + } else if (long.class == javaClass) { + long[] array = (long[]) value.asObject(); + int length = array.length; + BinaryStreamUtils.writeVarInt(output, length); + for (int i = 0; i < length; i++) { + serialize(v.update(array[i]), config, baseColumn, output); + } + } else if (float.class == javaClass) { + float[] array = (float[]) value.asObject(); + int length = array.length; + BinaryStreamUtils.writeVarInt(output, length); + for (int i = 0; i < length; i++) { + serialize(v.update(array[i]), config, baseColumn, output); + } + } else if (double.class == javaClass) { + double[] array = (double[]) value.asObject(); + int length = array.length; + BinaryStreamUtils.writeVarInt(output, length); + for (int i = 0; i < length; i++) { + serialize(v.update(array[i]), config, baseColumn, output); + } + } else { + throw new IllegalArgumentException("Unsupported primitive type: " + javaClass); + } + } + } + + private ClickHouseValue readArray(ClickHouseValue ref, ClickHouseConfig config, ClickHouseColumn nestedColumn, + ClickHouseColumn baseColumn, InputStream input, int length, int level) throws IOException { + Class javaClass = baseColumn.getDataType().getPrimitiveClass(); + if (level > 1 || !javaClass.isPrimitive()) { + Object[] array = (Object[]) ClickHouseValues.createPrimitiveArray(javaClass, length, level); + for (int i = 0; i < length; i++) { + array[i] = deserialize(null, config, nestedColumn, input).asObject(); + } + ref.update(array); + } else { + if (byte.class == javaClass) { + byte[] array = new byte[length]; + for (int i = 0; i < length; i++) { + array[i] = deserialize(null, config, baseColumn, input).asByte(); + } + ref.update(array); + } else if (short.class == javaClass) { + short[] array = new short[length]; + for (int i = 0; i < length; i++) { + array[i] = deserialize(null, config, baseColumn, input).asShort(); + } + ref.update(array); + } else if (int.class == javaClass) { + int[] array = new int[length]; + for (int i = 0; i < length; i++) { + array[i] = deserialize(null, config, baseColumn, input).asInteger(); + } + ref.update(array); + } else if (long.class == javaClass) { + long[] array = new long[length]; + for (int i = 0; i < length; i++) { + array[i] = deserialize(null, config, baseColumn, input).asLong(); + } + ref.update(array); + } else if (float.class == javaClass) { + float[] array = new float[length]; + for (int i = 0; i < length; i++) { + array[i] = deserialize(null, config, baseColumn, input).asFloat(); + } + ref.update(array); + } else if (double.class == javaClass) { + double[] array = new double[length]; + for (int i = 0; i < length; i++) { + array[i] = deserialize(null, config, baseColumn, input).asDouble(); + } + ref.update(array); + } else { + throw new IllegalArgumentException("Unsupported primitive type: " + javaClass); + } + } + + return ref; + } + private final Map> deserializers; private final Map> serializers; @@ -43,160 +165,202 @@ private MappedFunctions() { // enum and numbers buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseByteValue.of(r, BinaryStreamUtils.readInt8(i)), - (v, c, o) -> BinaryStreamUtils.writeInt8(o, v.asByte()), ClickHouseDataType.Enum, + (r, f, c, i) -> ClickHouseByteValue.of(r, BinaryStreamUtils.readInt8(i)), + (v, f, c, o) -> BinaryStreamUtils.writeInt8(o, v.asByte()), ClickHouseDataType.Enum, ClickHouseDataType.Enum8, ClickHouseDataType.Int8); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseShortValue.of(r, BinaryStreamUtils.readUnsignedInt8(i)), - (v, c, o) -> BinaryStreamUtils.writeUnsignedInt8(o, v.asInteger()), ClickHouseDataType.UInt8); + (r, f, c, i) -> ClickHouseShortValue.of(r, BinaryStreamUtils.readUnsignedInt8(i)), + (v, f, c, o) -> BinaryStreamUtils.writeUnsignedInt8(o, v.asInteger()), ClickHouseDataType.UInt8); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseShortValue.of(r, BinaryStreamUtils.readInt16(i)), - (v, c, o) -> BinaryStreamUtils.writeInt16(o, v.asShort()), ClickHouseDataType.Enum16, + (r, f, c, i) -> ClickHouseShortValue.of(r, BinaryStreamUtils.readInt16(i)), + (v, f, c, o) -> BinaryStreamUtils.writeInt16(o, v.asShort()), ClickHouseDataType.Enum16, ClickHouseDataType.Int16); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseIntegerValue.of(r, BinaryStreamUtils.readUnsignedInt16(i)), - (v, c, o) -> BinaryStreamUtils.writeUnsignedInt16(o, v.asInteger()), ClickHouseDataType.UInt16); + (r, f, c, i) -> ClickHouseIntegerValue.of(r, BinaryStreamUtils.readUnsignedInt16(i)), + (v, f, c, o) -> BinaryStreamUtils.writeUnsignedInt16(o, v.asInteger()), ClickHouseDataType.UInt16); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseIntegerValue.of(r, BinaryStreamUtils.readInt32(i)), - (v, c, o) -> BinaryStreamUtils.writeInt32(o, v.asInteger()), ClickHouseDataType.Int32); + (r, f, c, i) -> ClickHouseIntegerValue.of(r, BinaryStreamUtils.readInt32(i)), + (v, f, c, o) -> BinaryStreamUtils.writeInt32(o, v.asInteger()), ClickHouseDataType.Int32); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseLongValue.of(r, false, BinaryStreamUtils.readUnsignedInt32(i)), - (v, c, o) -> BinaryStreamUtils.writeUnsignedInt32(o, v.asLong()), ClickHouseDataType.UInt32); + (r, f, c, i) -> ClickHouseLongValue.of(r, false, BinaryStreamUtils.readUnsignedInt32(i)), + (v, f, c, o) -> BinaryStreamUtils.writeUnsignedInt32(o, v.asLong()), ClickHouseDataType.UInt32); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseLongValue.of(r, false, BinaryStreamUtils.readInt64(i)), - (v, c, o) -> BinaryStreamUtils.writeInt64(o, v.asLong()), ClickHouseDataType.IntervalYear, + (r, f, c, i) -> ClickHouseLongValue.of(r, false, BinaryStreamUtils.readInt64(i)), + (v, f, c, o) -> BinaryStreamUtils.writeInt64(o, v.asLong()), ClickHouseDataType.IntervalYear, ClickHouseDataType.IntervalQuarter, ClickHouseDataType.IntervalMonth, ClickHouseDataType.IntervalWeek, ClickHouseDataType.IntervalDay, ClickHouseDataType.IntervalHour, ClickHouseDataType.IntervalMinute, ClickHouseDataType.IntervalSecond, ClickHouseDataType.Int64); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseLongValue.of(r, true, BinaryStreamUtils.readInt64(i)), - (v, c, o) -> BinaryStreamUtils.writeInt64(o, v.asLong()), ClickHouseDataType.UInt64); + (r, f, c, i) -> ClickHouseLongValue.of(r, true, BinaryStreamUtils.readInt64(i)), + (v, f, c, o) -> BinaryStreamUtils.writeInt64(o, v.asLong()), ClickHouseDataType.UInt64); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseBigIntegerValue.of(r, BinaryStreamUtils.readInt128(i)), - (v, c, o) -> BinaryStreamUtils.writeInt128(o, v.asBigInteger()), ClickHouseDataType.Int128); + (r, f, c, i) -> ClickHouseBigIntegerValue.of(r, BinaryStreamUtils.readInt128(i)), + (v, f, c, o) -> BinaryStreamUtils.writeInt128(o, v.asBigInteger()), ClickHouseDataType.Int128); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseBigIntegerValue.of(r, BinaryStreamUtils.readUnsignedInt128(i)), - (v, c, o) -> BinaryStreamUtils.writeUnsignedInt128(o, v.asBigInteger()), + (r, f, c, i) -> ClickHouseBigIntegerValue.of(r, BinaryStreamUtils.readUnsignedInt128(i)), + (v, f, c, o) -> BinaryStreamUtils.writeUnsignedInt128(o, v.asBigInteger()), ClickHouseDataType.UInt128); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseBigIntegerValue.of(r, BinaryStreamUtils.readInt256(i)), - (v, c, o) -> BinaryStreamUtils.writeInt256(o, v.asBigInteger()), ClickHouseDataType.Int256); + (r, f, c, i) -> ClickHouseBigIntegerValue.of(r, BinaryStreamUtils.readInt256(i)), + (v, f, c, o) -> BinaryStreamUtils.writeInt256(o, v.asBigInteger()), ClickHouseDataType.Int256); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseBigIntegerValue.of(r, BinaryStreamUtils.readUnsignedInt256(i)), - (v, c, o) -> BinaryStreamUtils.writeUnsignedInt256(o, v.asBigInteger()), + (r, f, c, i) -> ClickHouseBigIntegerValue.of(r, BinaryStreamUtils.readUnsignedInt256(i)), + (v, f, c, o) -> BinaryStreamUtils.writeUnsignedInt256(o, v.asBigInteger()), ClickHouseDataType.UInt256); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseFloatValue.of(r, BinaryStreamUtils.readFloat32(i)), - (v, c, o) -> BinaryStreamUtils.writeFloat32(o, v.asFloat()), ClickHouseDataType.Float32); + (r, f, c, i) -> ClickHouseFloatValue.of(r, BinaryStreamUtils.readFloat32(i)), + (v, f, c, o) -> BinaryStreamUtils.writeFloat32(o, v.asFloat()), ClickHouseDataType.Float32); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseDoubleValue.of(r, BinaryStreamUtils.readFloat64(i)), - (v, c, o) -> BinaryStreamUtils.writeFloat64(o, v.asDouble()), ClickHouseDataType.Float64); + (r, f, c, i) -> ClickHouseDoubleValue.of(r, BinaryStreamUtils.readFloat64(i)), + (v, f, c, o) -> BinaryStreamUtils.writeFloat64(o, v.asDouble()), ClickHouseDataType.Float64); // decimals buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseBigDecimalValue.of(r, + (r, f, c, i) -> ClickHouseBigDecimalValue.of(r, BinaryStreamUtils.readDecimal(i, c.getPrecision(), c.getScale())), - (v, c, o) -> BinaryStreamUtils.writeDecimal(o, v.asBigDecimal(c.getScale()), c.getPrecision(), + (v, f, c, o) -> BinaryStreamUtils.writeDecimal(o, v.asBigDecimal(c.getScale()), c.getPrecision(), c.getScale()), ClickHouseDataType.Decimal); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseBigDecimalValue.of(r, BinaryStreamUtils.readDecimal32(i, c.getScale())), - (v, c, o) -> BinaryStreamUtils.writeDecimal32(o, v.asBigDecimal(c.getScale()), c.getScale()), + (r, f, c, i) -> ClickHouseBigDecimalValue.of(r, BinaryStreamUtils.readDecimal32(i, c.getScale())), + (v, f, c, o) -> BinaryStreamUtils.writeDecimal32(o, v.asBigDecimal(c.getScale()), c.getScale()), ClickHouseDataType.Decimal32); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseBigDecimalValue.of(r, BinaryStreamUtils.readDecimal64(i, c.getScale())), - (v, c, o) -> BinaryStreamUtils.writeDecimal64(o, v.asBigDecimal(c.getScale()), c.getScale()), + (r, f, c, i) -> ClickHouseBigDecimalValue.of(r, BinaryStreamUtils.readDecimal64(i, c.getScale())), + (v, f, c, o) -> BinaryStreamUtils.writeDecimal64(o, v.asBigDecimal(c.getScale()), c.getScale()), ClickHouseDataType.Decimal64); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseBigDecimalValue.of(r, BinaryStreamUtils.readDecimal128(i, c.getScale())), - (v, c, o) -> BinaryStreamUtils.writeDecimal128(o, v.asBigDecimal(c.getScale()), c.getScale()), + (r, f, c, i) -> ClickHouseBigDecimalValue.of(r, BinaryStreamUtils.readDecimal128(i, c.getScale())), + (v, f, c, o) -> BinaryStreamUtils.writeDecimal128(o, v.asBigDecimal(c.getScale()), c.getScale()), ClickHouseDataType.Decimal128); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseBigDecimalValue.of(r, BinaryStreamUtils.readDecimal256(i, c.getScale())), - (v, c, o) -> BinaryStreamUtils.writeDecimal256(o, v.asBigDecimal(c.getScale()), c.getScale()), + (r, f, c, i) -> ClickHouseBigDecimalValue.of(r, BinaryStreamUtils.readDecimal256(i, c.getScale())), + (v, f, c, o) -> BinaryStreamUtils.writeDecimal256(o, v.asBigDecimal(c.getScale()), c.getScale()), ClickHouseDataType.Decimal256); // date, time, datetime and IPs buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseDateValue.of(r, BinaryStreamUtils.readDate(i)), - (v, c, o) -> BinaryStreamUtils.writeDate(o, v.asDate()), ClickHouseDataType.Date); - buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseDateValue.of(r, BinaryStreamUtils.readDate32(i)), - (v, c, o) -> BinaryStreamUtils.writeDate(o, v.asDate()), ClickHouseDataType.Date32); - buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseDateTimeValue.of(r, - (c.getScale() > 0 ? BinaryStreamUtils.readDateTime64(i, c.getScale()) - : BinaryStreamUtils.readDateTime(i)), - c.getScale()), - (v, c, o) -> BinaryStreamUtils.writeDateTime(o, v.asDateTime(), c.getScale()), + (r, f, c, i) -> ClickHouseDateValue.of(r, BinaryStreamUtils.readDate(i, f.getServerTimeZone())), + (v, f, c, o) -> BinaryStreamUtils.writeDate(o, v.asDate(), f.getServerTimeZone()), + ClickHouseDataType.Date); + buildMappings(deserializers, serializers, + (r, f, c, i) -> ClickHouseDateValue.of(r, BinaryStreamUtils.readDate32(i, f.getServerTimeZone())), + (v, f, c, o) -> BinaryStreamUtils.writeDate(o, v.asDate(), f.getServerTimeZone()), + ClickHouseDataType.Date32); + buildMappings(deserializers, serializers, (r, f, c, i) -> c.getTimeZone() == null + ? ClickHouseDateTimeValue.of(r, + (c.getScale() > 0 ? BinaryStreamUtils.readDateTime64(i, c.getScale(), f.getServerTimeZone()) + : BinaryStreamUtils.readDateTime(i, f.getServerTimeZone())), + c.getScale()) + : ClickHouseOffsetDateTimeValue.of(r, + (c.getScale() > 0 ? BinaryStreamUtils.readDateTime64(i, c.getScale(), c.getTimeZone()) + : BinaryStreamUtils.readDateTime(i, c.getTimeZone())), + c.getScale(), c.getTimeZone()), + (v, f, c, o) -> BinaryStreamUtils.writeDateTime(o, v.asDateTime(), c.getScale(), + c.getTimeZoneOrDefault(f.getServerTimeZone())), ClickHouseDataType.DateTime); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseDateTimeValue.of(r, BinaryStreamUtils.readDateTime(i), 0), - (v, c, o) -> BinaryStreamUtils.writeDateTime32(o, v.asDateTime()), ClickHouseDataType.DateTime32); - buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseDateTimeValue.of(r, BinaryStreamUtils.readDateTime64(i, c.getScale()), - c.getScale()), - (v, c, o) -> BinaryStreamUtils.writeDateTime64(o, v.asDateTime(), c.getScale()), + (r, f, c, i) -> c.getTimeZone() == null + ? ClickHouseDateTimeValue.of(r, BinaryStreamUtils.readDateTime(i, f.getServerTimeZone()), 0) + : ClickHouseOffsetDateTimeValue.of(r, BinaryStreamUtils.readDateTime(i, c.getTimeZone()), 0, + c.getTimeZone()), + (v, f, c, o) -> BinaryStreamUtils.writeDateTime32(o, v.asDateTime(), + c.getTimeZoneOrDefault(f.getServerTimeZone())), + ClickHouseDataType.DateTime32); + buildMappings(deserializers, serializers, + (r, f, c, i) -> c.getTimeZone() == null ? ClickHouseDateTimeValue.of(r, + BinaryStreamUtils.readDateTime64(i, c.getScale(), f.getServerTimeZone()), c.getScale()) + : ClickHouseOffsetDateTimeValue.of(r, + BinaryStreamUtils.readDateTime64(i, c.getScale(), c.getTimeZone()), c.getScale(), + c.getTimeZone()), + (v, f, c, o) -> BinaryStreamUtils.writeDateTime64(o, v.asDateTime(), c.getScale(), + c.getTimeZoneOrDefault(f.getServerTimeZone())), ClickHouseDataType.DateTime64); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseIpv4Value.of(r, BinaryStreamUtils.readInet4Address(i)), - (v, c, o) -> BinaryStreamUtils.writeInet4Address(o, v.asInet4Address()), ClickHouseDataType.IPv4); + (r, f, c, i) -> ClickHouseIpv4Value.of(r, BinaryStreamUtils.readInet4Address(i)), + (v, f, c, o) -> BinaryStreamUtils.writeInet4Address(o, v.asInet4Address()), + ClickHouseDataType.IPv4); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseIpv6Value.of(r, BinaryStreamUtils.readInet6Address(i)), - (v, c, o) -> BinaryStreamUtils.writeInet6Address(o, v.asInet6Address()), ClickHouseDataType.IPv6); + (r, f, c, i) -> ClickHouseIpv6Value.of(r, BinaryStreamUtils.readInet6Address(i)), + (v, f, c, o) -> BinaryStreamUtils.writeInet6Address(o, v.asInet6Address()), + ClickHouseDataType.IPv6); // string and uuid buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseStringValue.of(r, BinaryStreamUtils.readFixedString(i, c.getPrecision())), - (v, c, o) -> BinaryStreamUtils.writeFixedString(o, v.asString(c.getPrecision()), c.getPrecision()), + (r, f, c, i) -> ClickHouseStringValue.of(r, BinaryStreamUtils.readFixedString(i, c.getPrecision())), + (v, f, c, o) -> BinaryStreamUtils.writeFixedString(o, v.asString(c.getPrecision()), + c.getPrecision()), ClickHouseDataType.FixedString); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseStringValue.of(r, BinaryStreamUtils.readString(i)), - (v, c, o) -> BinaryStreamUtils.writeString(o, v.asString()), ClickHouseDataType.String); + (r, f, c, i) -> ClickHouseStringValue.of(r, BinaryStreamUtils.readString(i)), + (v, f, c, o) -> BinaryStreamUtils.writeString(o, v.asString()), ClickHouseDataType.String); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseUuidValue.of(r, BinaryStreamUtils.readUuid(i)), - (v, c, o) -> BinaryStreamUtils.writeUuid(o, v.asUuid()), ClickHouseDataType.UUID); + (r, f, c, i) -> ClickHouseUuidValue.of(r, BinaryStreamUtils.readUuid(i)), + (v, f, c, o) -> BinaryStreamUtils.writeUuid(o, v.asUuid()), ClickHouseDataType.UUID); // geo types buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseGeoPointValue.of(r, BinaryStreamUtils.readGeoPoint(i)), (v, c, o) -> { - }, ClickHouseDataType.Point); + (r, f, c, i) -> ClickHouseGeoPointValue.of(r, BinaryStreamUtils.readGeoPoint(i)), + (v, f, c, o) -> BinaryStreamUtils.writeGeoPoint(o, v.asObject(double[].class)), + ClickHouseDataType.Point); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseGeoRingValue.of(r, BinaryStreamUtils.readGeoRing(i)), (v, c, o) -> { - }, ClickHouseDataType.Ring); + (r, f, c, i) -> ClickHouseGeoRingValue.of(r, BinaryStreamUtils.readGeoRing(i)), + (v, f, c, o) -> BinaryStreamUtils.writeGeoRing(o, v.asObject(double[][].class)), + ClickHouseDataType.Ring); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseGeoPolygonValue.of(r, BinaryStreamUtils.readGeoPolygon(i)), (v, c, o) -> { - }, ClickHouseDataType.Polygon); + (r, f, c, i) -> ClickHouseGeoPolygonValue.of(r, BinaryStreamUtils.readGeoPolygon(i)), + (v, f, c, o) -> BinaryStreamUtils.writeGeoPolygon(o, v.asObject(double[][][].class)), + ClickHouseDataType.Polygon); buildMappings(deserializers, serializers, - (r, c, i) -> ClickHouseGeoMultiPolygonValue.of(r, BinaryStreamUtils.readGeoMultiPolygon(i)), - (v, c, o) -> { - }, ClickHouseDataType.MultiPolygon); + (r, f, c, i) -> ClickHouseGeoMultiPolygonValue.of(r, BinaryStreamUtils.readGeoMultiPolygon(i)), + (v, f, c, o) -> BinaryStreamUtils.writeGeoMultiPolygon(o, v.asObject(double[][][][].class)), + ClickHouseDataType.MultiPolygon); // advanced types - buildMappings(deserializers, serializers, (r, c, i) -> { - int size = BinaryStreamUtils.readVarInt(i); - ClickHouseColumn nestedColumn = c.getNestedColumns().get(0); - // TODO optimize primitive array - Object[] arr = (Object[]) Array.newInstance(nestedColumn.getDataType().getJavaClass(), size); - for (int k = 0; k < size; k++) { - arr[k] = deserialize(nestedColumn, null, i).asObject(); + buildMappings(deserializers, serializers, (r, f, c, i) -> { + if (c.getAggregateFunction() == ClickHouseAggregateFunction.groupBitmap) { + return ClickHouseBitmapValue + .of(BinaryStreamUtils.readBitmap(i, c.getNestedColumns().get(0).getDataType())); + } + return null; + }, (v, f, c, o) -> { + if (c.getAggregateFunction() == ClickHouseAggregateFunction.groupBitmap) { + BinaryStreamUtils.writeBitmap(o, v.asObject(ClickHouseBitmap.class)); } - return ClickHouseArrayValue.of(r, arr); - }, (v, c, o) -> { - }, ClickHouseDataType.Array); - buildMappings(deserializers, serializers, (r, c, i) -> { + }, ClickHouseDataType.AggregateFunction); + buildMappings(deserializers, serializers, (r, f, c, i) -> { + int length = BinaryStreamUtils.readVarInt(i); + if (r == null) { + r = ClickHouseValues.newValue(c); + } + return readArray(r, f, c.getNestedColumns().get(0), c.getArrayBaseColumn(), i, length, + c.getArrayNestedLevel()); + }, this::writeArray, ClickHouseDataType.Array); + buildMappings(deserializers, serializers, (r, f, c, i) -> { Map map = new LinkedHashMap<>(); ClickHouseColumn keyCol = c.getKeyInfo(); ClickHouseColumn valCol = c.getValueInfo(); for (int k = 0, len = BinaryStreamUtils.readVarInt(i); k < len; k++) { - map.put(deserialize(keyCol, null, i).asObject(), deserialize(valCol, null, i).asObject()); + map.put(deserialize(null, f, keyCol, i).asObject(), deserialize(null, f, valCol, i).asObject()); + } + return ClickHouseMapValue.of(map, valCol.getDataType().getObjectClass(), + valCol.getDataType().getObjectClass()); + }, (v, f, c, o) -> { + Map map = v.asMap(); + BinaryStreamUtils.writeVarInt(o, map.size()); + if (!map.isEmpty()) { + ClickHouseColumn keyCol = c.getKeyInfo(); + ClickHouseColumn valCol = c.getValueInfo(); + ClickHouseValue kVal = ClickHouseValues.newValue(keyCol); + ClickHouseValue vVal = ClickHouseValues.newValue(valCol); + for (Entry e : map.entrySet()) { + serialize(kVal.update(e.getKey()), f, keyCol, o); + serialize(vVal.update(e.getValue()), f, valCol, o); + } } - return ClickHouseMapValue.of(map, valCol.getDataType().getJavaClass(), - valCol.getDataType().getJavaClass()); - }, (v, c, o) -> { }, ClickHouseDataType.Map); - buildMappings(deserializers, serializers, (r, c, i) -> { + buildMappings(deserializers, serializers, (r, f, c, i) -> { int count = c.getNestedColumns().size(); String[] names = new String[count]; Object[][] values = new Object[count][]; @@ -206,26 +370,48 @@ private MappedFunctions() { int k = BinaryStreamUtils.readVarInt(i); Object[] nvalues = new Object[k]; for (int j = 0; j < k; j++) { - nvalues[j] = deserialize(col, null, i).asObject(); + nvalues[j] = deserialize(null, f, col, i).asObject(); } values[l++] = nvalues; } return ClickHouseNestedValue.of(r, c.getNestedColumns(), values); - }, (v, c, o) -> { + }, (v, f, c, o) -> { + Object[][] values = (Object[][]) v.asObject(); + int l = 0; + for (ClickHouseColumn col : c.getNestedColumns()) { + Object[] nvalues = values[l++]; + int k = nvalues.length; + ClickHouseValue nv = ClickHouseValues.newValue(col); + BinaryStreamUtils.writeVarInt(o, k); + for (int j = 0; j < k; j++) { + serialize(nv.update(nvalues[j]), f, col, o); + } + } }, ClickHouseDataType.Nested); - buildMappings(deserializers, serializers, (r, c, i) -> { + buildMappings(deserializers, serializers, (r, f, c, i) -> { List tupleValues = new ArrayList<>(c.getNestedColumns().size()); for (ClickHouseColumn col : c.getNestedColumns()) { - tupleValues.add(deserialize(col, null, i).asObject()); + tupleValues.add(deserialize(null, f, col, i).asObject()); } return ClickHouseTupleValue.of(r, tupleValues); - }, (v, c, o) -> { + }, (v, f, c, o) -> { + List tupleValues = v.asTuple(); + Iterator tupleIterator = tupleValues.iterator(); + for (ClickHouseColumn col : c.getNestedColumns()) { + // FIXME tooooo slow + ClickHouseValue tv = ClickHouseValues.newValue(col); + if (tupleIterator.hasNext()) { + serialize(tv.update(tupleIterator.next()), f, col, o); + } else { + serialize(tv, f, col, o); + } + } }, ClickHouseDataType.Tuple); } @SuppressWarnings("unchecked") - public ClickHouseValue deserialize(ClickHouseColumn column, ClickHouseValue ref, InputStream input) - throws IOException { + public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseConfig config, ClickHouseColumn column, + InputStream input) throws IOException { if (column.isNullable() && BinaryStreamUtils.readNull(input)) { return ref == null ? ClickHouseValues.newValue(column) : ref.resetToNullOrEmpty(); } @@ -235,11 +421,12 @@ public ClickHouseValue deserialize(ClickHouseColumn column, ClickHouseValue ref, if (func == null) { throw new IllegalArgumentException(ERROR_UNKNOWN_DATA_TYPE + column.getDataType().name()); } - return func.deserialize(ref, column, input); + return func.deserialize(ref, config, column, input); } @SuppressWarnings("unchecked") - public void serialize(ClickHouseColumn column, ClickHouseValue value, OutputStream output) throws IOException { + public void serialize(ClickHouseValue value, ClickHouseConfig config, ClickHouseColumn column, + OutputStream output) throws IOException { if (column.isNullable()) { // always false for geo types, and Array, Nested, Map and Tuple etc. if (value.isNullOrEmpty()) { BinaryStreamUtils.writeNull(output); @@ -254,7 +441,7 @@ public void serialize(ClickHouseColumn column, ClickHouseValue value, OutputStre if (func == null) { throw new IllegalArgumentException(ERROR_UNKNOWN_DATA_TYPE + column.getDataType().name()); } - func.serialize(value, column, output); + func.serialize(value, config, column, output); } } @@ -262,44 +449,49 @@ public static MappedFunctions getMappedFunctions() { return MappedFunctions.instance; } + // TODO this is where ASM should come into play... private class Records implements Iterator { - private final Supplier factory; - private ClickHouseValue[] values; + private final Supplier factory; + private ClickHouseSimpleRecord record; Records() { int size = columns.size(); if (config.isReuseValueWrapper()) { - values = new ClickHouseValue[size]; - factory = () -> values; + ClickHouseValue[] values = new ClickHouseValue[size]; + record = new ClickHouseSimpleRecord(columns, values); + factory = () -> record; } else { - factory = () -> new ClickHouseValue[size]; + factory = () -> new ClickHouseSimpleRecord(columns, new ClickHouseValue[size]); } } - void readNextRow() { + ClickHouseRecord readNextRow() { int index = 0; int size = columns.size(); - values = factory.get(); + ClickHouseSimpleRecord currentRow = factory.get(); + ClickHouseValue[] values = currentRow.getValues(); ClickHouseColumn column = null; try { MappedFunctions m = getMappedFunctions(); for (; index < size; index++) { column = columns.get(index); - values[index] = m.deserialize(column, values[index], input); + values[index] = m.deserialize(values[index], config, column, input); } } catch (EOFException e) { if (index == 0) { // end of the stream, which is fine values = null; } else { - throw new IllegalStateException( + throw new UncheckedIOException( ClickHouseUtils.format("Reached end of the stream when reading column #%d(total %d): %s", index + 1, size, column), e); } } catch (IOException e) { - throw new IllegalStateException( + throw new UncheckedIOException( ClickHouseUtils.format("Failed to read column #%d(total %d): %s", index + 1, size, column), e); } + + return currentRow; } @Override @@ -307,38 +499,17 @@ public boolean hasNext() { try { return input.available() > 0; } catch (IOException e) { - throw new IllegalStateException(e); + throw new UncheckedIOException(e); } } @Override public ClickHouseRecord next() { - readNextRow(); - - return new ClickHouseRecord() { - @Override - public int size() { - return values.length; - } - - @Override - public ClickHouseValue getValue(int index) throws IOException { - return values[index]; - } - - @Override - public ClickHouseValue getValue(String columnName) throws IOException { - int index = 0; - for (ClickHouseColumn c : columns) { - if (c.getColumnName().equals(columnName)) { - getValue(index); - } - index++; - } + if (!hasNext()) { + throw new NoSuchElementException("No more record"); + } - throw new IllegalArgumentException("Not able to find a column named: " + columnName); - } - }; + return readNextRow(); } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseSimpleRecord.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseSimpleRecord.java new file mode 100644 index 000000000..466e9c4cf --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseSimpleRecord.java @@ -0,0 +1,92 @@ +package com.clickhouse.client.data; + +import java.util.Collections; +import java.util.List; + +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.ClickHouseRecord; +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.ClickHouseValue; + +/** + * Default implementation of {@link com.clickhouse.client.ClickHouseRecord}, + * which is simply a combination of list of columns and array of values. + */ +public class ClickHouseSimpleRecord implements ClickHouseRecord { + public static final ClickHouseSimpleRecord EMPTY = new ClickHouseSimpleRecord(Collections.emptyList(), + new ClickHouseValue[0]); + + private final List columns; + private ClickHouseValue[] values; + + /** + * Creates a record object to wrap given values. + * + * @param columns non-null list of columns + * @param values non-null array of values + * @return record + */ + public static ClickHouseRecord of(List columns, ClickHouseValue[] values) { + if (columns == null || values == null) { + throw new IllegalArgumentException("Non-null columns and values are required"); + } else if (columns.size() != values.length) { + throw new IllegalArgumentException(ClickHouseUtils.format( + "Mismatched count: we have %d columns but we got %d values", columns.size(), values.length)); + } else if (values.length == 0) { + return EMPTY; + } + + return new ClickHouseSimpleRecord(columns, values); + } + + protected ClickHouseSimpleRecord(List columns, ClickHouseValue[] values) { + this.columns = columns; + this.values = values; + } + + protected List getColumns() { + return columns; + } + + protected ClickHouseValue[] getValues() { + return values; + } + + protected void update(ClickHouseValue[] values) { + this.values = values; + } + + protected void update(Object[] values) { + int len = values != null ? values.length : 0; + for (int i = 0, size = this.values.length; i < size; i++) { + if (i < len) { + this.values[i].update(values[i]); + } else { + this.values[i].resetToNullOrEmpty(); + } + } + } + + @Override + public int size() { + return values.length; + } + + @Override + public ClickHouseValue getValue(int index) { + return values[index]; + } + + @Override + public ClickHouseValue getValue(String name) { + int index = 0; + for (ClickHouseColumn c : columns) { + if (c.getColumnName().equalsIgnoreCase(name)) { + return getValue(index); + } + index++; + } + + throw new IllegalArgumentException(ClickHouseUtils.format("Unable to find column [%s]", name)); + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseSimpleResponse.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseSimpleResponse.java new file mode 100644 index 000000000..73b4dd71e --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseSimpleResponse.java @@ -0,0 +1,179 @@ +package com.clickhouse.client.data; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.ClickHouseRecord; +import com.clickhouse.client.ClickHouseResponse; +import com.clickhouse.client.ClickHouseResponseSummary; +import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.client.ClickHouseValues; + +/** + * A simple response built on top of two lists: columns and records. + */ +public class ClickHouseSimpleResponse implements ClickHouseResponse { + public static final ClickHouseSimpleResponse EMPTY = new ClickHouseSimpleResponse(Collections.emptyList(), + new ClickHouseValue[0][], ClickHouseResponseSummary.EMPTY); + + /** + * Creates a response object using columns definition and raw values. + * + * @param columns list of columns + * @param values raw values, which may or may not be null + * @return response object + */ + public static ClickHouseResponse of(List columns, Object[][] values) { + return of(columns, values, null); + } + + /** + * Creates a response object using columns definition and raw values. + * + * @param columns list of columns + * @param values raw values, which may or may not be null + * @return response object + */ + public static ClickHouseResponse of(List columns, Object[][] values, + ClickHouseResponseSummary summary) { + if (columns == null || columns.isEmpty()) { + return EMPTY; + } + + int size = columns.size(); + int len = values != null ? values.length : 0; + + ClickHouseValue[][] wrappedValues = new ClickHouseValue[len][]; + if (len > 0) { + ClickHouseValue[] templates = new ClickHouseValue[size]; + for (int i = 0; i < size; i++) { + templates[i] = ClickHouseValues.newValue(columns.get(i)); + } + + for (int i = 0; i < len; i++) { + Object[] input = values[i]; + int count = input != null ? input.length : 0; + ClickHouseValue[] v = new ClickHouseValue[size]; + for (int j = 0; j < size; j++) { + v[j] = templates[j].copy().update(j < count ? input[j] : null); + } + wrappedValues[i] = v; + } + } + + return new ClickHouseSimpleResponse(columns, wrappedValues, summary); + } + + /** + * Creates a response object by copying columns and values from the given one. + * Same as {@code of(response, null)}. + * + * @param response response to copy + * @return new response object + */ + public static ClickHouseResponse of(ClickHouseResponse response) { + return of(response, null); + } + + /** + * Creates a response object by copying columns and values from the given one. + * You should never use this method against a large response, because it will + * load everything into memory. Worse than that, when {@code func} is not null, + * it will be applied to every single row, which is going to be slow when + * original response contains many records. + * + * @param response response to copy + * @param func optinal function to update value by column index + * @return new response object + */ + public static ClickHouseResponse of(ClickHouseResponse response, ClickHouseRecordTransformer func) { + if (response == null) { + return EMPTY; + } else if (response instanceof ClickHouseSimpleResponse) { + return response; + } + + List columns = response.getColumns(); + int size = columns.size(); + List records = new LinkedList<>(); + int rowIndex = 0; + for (ClickHouseRecord r : response.records()) { + ClickHouseValue[] values = new ClickHouseValue[size]; + for (int i = 0; i < size; i++) { + values[i] = r.getValue(i).copy(); + } + + ClickHouseRecord rec = ClickHouseSimpleRecord.of(columns, values); + if (func != null) { + func.update(rowIndex, rec); + } + records.add(rec); + } + + return new ClickHouseSimpleResponse(response.getColumns(), records, response.getSummary()); + } + + private final List columns; + // better to use simple ClickHouseRecord as template along with raw values + private final List records; + private final ClickHouseResponseSummary summary; + + private boolean isClosed; + + protected ClickHouseSimpleResponse(List columns, List records, + ClickHouseResponseSummary summary) { + this.columns = columns; + this.records = Collections.unmodifiableList(records); + this.summary = summary != null ? summary : ClickHouseResponseSummary.EMPTY; + } + + protected ClickHouseSimpleResponse(List columns, ClickHouseValue[][] values, + ClickHouseResponseSummary summary) { + this.columns = columns; + + int len = values.length; + List list = new ArrayList<>(len); + for (int i = 0; i < len; i++) { + list.add(new ClickHouseSimpleRecord(columns, values[i])); + } + + this.records = Collections.unmodifiableList(list); + + this.summary = summary != null ? summary : ClickHouseResponseSummary.EMPTY; + } + + @Override + public List getColumns() { + return columns; + } + + @Override + public ClickHouseResponseSummary getSummary() { + return summary; + } + + @Override + public InputStream getInputStream() { + throw new UnsupportedOperationException("An in-memory response does not have input stream"); + } + + @Override + public Iterable records() { + return records; + } + + @Override + public void close() { + // nothing to close + isClosed = true; + } + + @Override + public boolean isClosed() { + return isClosed; + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseStreamResponse.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseStreamResponse.java new file mode 100644 index 000000000..d33eabd88 --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseStreamResponse.java @@ -0,0 +1,146 @@ +package com.clickhouse.client.data; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.ClickHouseConfig; +import com.clickhouse.client.ClickHouseDataProcessor; +import com.clickhouse.client.ClickHouseDataStreamFactory; +import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.client.ClickHouseRecord; +import com.clickhouse.client.ClickHouseResponse; +import com.clickhouse.client.ClickHouseResponseSummary; +import com.clickhouse.client.logging.Logger; +import com.clickhouse.client.logging.LoggerFactory; + +/** + * A stream response from server. + */ +public class ClickHouseStreamResponse implements ClickHouseResponse { + private static final Logger log = LoggerFactory.getLogger(ClickHouseStreamResponse.class); + + private static final long serialVersionUID = 2271296998310082447L; + + protected static final List defaultTypes = Collections + .singletonList(ClickHouseColumn.of("results", "Nullable(String)")); + + public static ClickHouseResponse of(ClickHouseConfig config, InputStream input) throws IOException { + return of(config, input, null, null, null); + } + + public static ClickHouseResponse of(ClickHouseConfig config, InputStream input, Map settings) + throws IOException { + return of(config, input, settings, null, null); + } + + public static ClickHouseResponse of(ClickHouseConfig config, InputStream input, List columns) + throws IOException { + return of(config, input, null, columns, null); + } + + public static ClickHouseResponse of(ClickHouseConfig config, InputStream input, Map settings, + List columns) throws IOException { + return of(config, input, settings, columns, null); + } + + public static ClickHouseResponse of(ClickHouseConfig config, InputStream input, Map settings, + List columns, ClickHouseResponseSummary summary) throws IOException { + return new ClickHouseStreamResponse(config, input, settings, columns, summary); + } + + protected final ClickHouseConfig config; + protected final transient InputStream input; + protected final transient ClickHouseDataProcessor processor; + protected final List columns; + protected final ClickHouseResponseSummary summary; + + private boolean isClosed; + + protected ClickHouseStreamResponse(ClickHouseConfig config, InputStream input, Map settings, + List columns, ClickHouseResponseSummary summary) throws IOException { + if (config == null || input == null) { + throw new IllegalArgumentException("Non-null configuration and input stream are required"); + } + + this.config = config; + this.input = input; + + boolean hasError = true; + try { + this.processor = ClickHouseDataStreamFactory.getInstance().getProcessor(config, input, null, settings, + columns); + this.columns = columns != null ? columns + : (processor != null ? processor.getColumns() : Collections.emptyList()); + hasError = false; + } finally { + if (hasError) { + // rude but safe + log.error("Failed to create stream response, closing input stream"); + try { + input.close(); + } catch (Exception e) { + // ignore + } + } + } + this.summary = summary != null ? summary : ClickHouseResponseSummary.EMPTY; + this.isClosed = hasError; + } + + @Override + public boolean isClosed() { + return isClosed; + } + + @Override + public void close() { + if (input != null) { + try { + log.debug("%d bytes skipped before closing input stream", input.skip(Long.MAX_VALUE)); + } catch (Exception e) { + // ignore + log.debug("Failed to skip reading input stream due to: %s", e.getMessage()); + } finally { + try { + input.close(); + } catch (Exception e) { + log.warn("Failed to close input stream", e); + } + isClosed = true; + } + } + } + + @Override + public List getColumns() { + return columns; + } + + public ClickHouseFormat getFormat() { + return this.config.getFormat(); + } + + @Override + public ClickHouseResponseSummary getSummary() { + return summary; + } + + @Override + public InputStream getInputStream() { + return input; + } + + @Override + public Iterable records() { + if (processor == null) { + throw new UnsupportedOperationException( + "No data processor available for deserialization, please consider to use getInputStream instead"); + } + + return processor.records(); + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseStringValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseStringValue.java index bcd082b49..c5e52d15e 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseStringValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseStringValue.java @@ -241,12 +241,12 @@ public ClickHouseStringValue update(double value) { @Override public ClickHouseValue update(BigInteger value) { - return set(String.valueOf(value)); + return set(value == null ? null : String.valueOf(value)); } @Override public ClickHouseValue update(BigDecimal value) { - return set(String.valueOf(value)); + return set(value == null ? null : String.valueOf(value)); } @Override diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTabSeparatedProcessor.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTabSeparatedProcessor.java index d480d4980..56441ae68 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTabSeparatedProcessor.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTabSeparatedProcessor.java @@ -15,13 +15,18 @@ import com.clickhouse.client.ClickHouseDataProcessor; import com.clickhouse.client.ClickHouseFormat; import com.clickhouse.client.ClickHouseRecord; +import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.ClickHouseValue; import com.clickhouse.client.data.tsv.ByteFragment; import com.clickhouse.client.data.tsv.StreamSplitter; public class ClickHouseTabSeparatedProcessor extends ClickHouseDataProcessor { - private static String[] toStringArray(ByteFragment headerFragment) { - ByteFragment[] split = headerFragment.split((byte) 0x09); + private static String[] toStringArray(ByteFragment headerFragment, byte delimitter) { + if (delimitter == (byte) 0) { + return new String[] { headerFragment.asString(true) }; + } + + ByteFragment[] split = headerFragment.split(delimitter); String[] array = new String[split.length]; for (int i = 0; i < split.length; i++) { array[i] = split[i].asString(true); @@ -57,7 +62,8 @@ public ClickHouseRecord next() { throw new NoSuchElementException("No more record"); } - ByteFragment[] currentCols = currentRow.split((byte) 0x09); + ByteFragment[] currentCols = colDelimitter != (byte) 0 ? currentRow.split(colDelimitter) + : new ByteFragment[] { currentRow }; readNextRow(); return new ClickHouseRecord() { @@ -72,32 +78,51 @@ public ClickHouseValue getValue(int index) { } @Override - public ClickHouseValue getValue(String columnName) throws IOException { + public ClickHouseValue getValue(String name) { int index = 0; for (ClickHouseColumn c : columns) { - if (c.getColumnName().equals(columnName)) { - getValue(index); + if (c.getColumnName().equalsIgnoreCase(name)) { + return getValue(index); } index++; } - throw new IllegalArgumentException("Not able to find a column named: " + columnName); + throw new IllegalArgumentException(ClickHouseUtils.format("Unable to find column [%s]", name)); } }; } } + private final byte rowDelimitter = (byte) 0x0A; + + // initialize in readColumns() + private byte colDelimitter; private StreamSplitter splitter; @Override public List readColumns() throws IOException { if (input == null) { return Collections.emptyList(); - } else if (!config.getFormat().hasHeader()) { + } + + ClickHouseFormat format = config.getFormat(); + if (!format.hasHeader()) { return DEFAULT_COLUMNS; } - this.splitter = new StreamSplitter(input, (byte) 0x0A, config.getMaxBufferSize()); + switch (config.getFormat()) { + case TSVWithNames: + case TSVWithNamesAndTypes: + case TabSeparatedWithNames: + case TabSeparatedWithNamesAndTypes: + colDelimitter = (byte) 0x09; + break; + default: + colDelimitter = (byte) 0; + break; + } + + this.splitter = new StreamSplitter(input, rowDelimitter, config.getMaxBufferSize()); ByteFragment headerFragment = this.splitter.next(); if (headerFragment == null) { @@ -108,15 +133,16 @@ public List readColumns() throws IOException { input.close(); throw new IllegalArgumentException("ClickHouse error: " + header); } - String[] cols = toStringArray(headerFragment); + String[] cols = toStringArray(headerFragment, colDelimitter); String[] types = null; - if (ClickHouseFormat.TabSeparatedWithNamesAndTypes == config.getFormat()) { + if (ClickHouseFormat.TSVWithNamesAndTypes == format + || ClickHouseFormat.TabSeparatedWithNamesAndTypes == format) { ByteFragment typesFragment = splitter.next(); if (typesFragment == null) { throw new IllegalArgumentException("ClickHouse response without column types"); } - types = toStringArray(typesFragment); + types = toStringArray(typesFragment, colDelimitter); } List list = new ArrayList<>(cols.length); @@ -132,7 +158,7 @@ public ClickHouseTabSeparatedProcessor(ClickHouseConfig config, InputStream inpu super(config, input, output, columns, settings); if (this.splitter == null && input != null) { - this.splitter = new StreamSplitter(input, (byte) 0x0A, config.getMaxBufferSize()); + this.splitter = new StreamSplitter(input, rowDelimitter, config.getMaxBufferSize()); } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTimeValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTimeValue.java deleted file mode 100644 index 4a135ef08..000000000 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTimeValue.java +++ /dev/null @@ -1,280 +0,0 @@ -package com.clickhouse.client.data; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; - -/** - * Wraper class of LocalTime. - */ -public class ClickHouseTimeValue extends ClickHouseObjectValue { - /** - * Create a new instance representing null value. - * - * @return new instance representing null value - */ - public static ClickHouseTimeValue ofNull() { - return ofNull(null); - } - - /** - * Update given value to null or create a new instance if {@code ref} is null. - * - * @param ref object to update, could be null - * @return same object as {@code ref} or a new instance if it's null - */ - public static ClickHouseTimeValue ofNull(ClickHouseValue ref) { - return ref instanceof ClickHouseTimeValue ? (ClickHouseTimeValue) ((ClickHouseTimeValue) ref).set(null) - : new ClickHouseTimeValue(null); - } - - /** - * Wrap the given value. - * - * @param value value - * @return object representing the value - */ - public static ClickHouseTimeValue of(LocalTime value) { - return of(null, value); - } - - /** - * Wrap the given value. - * - * @param secondOfDay second of day - * @return object representing the value - */ - public static ClickHouseTimeValue of(long secondOfDay) { - // what about nano second? - return of(null, LocalTime.ofSecondOfDay(secondOfDay)); - } - - /** - * Update value of the given object or create a new instance if {@code ref} is - * null. - * - * @param ref object to update, could be null - * @param value value - * @return same object as {@code ref} or a new instance if it's null - */ - public static ClickHouseTimeValue of(ClickHouseValue ref, LocalTime value) { - return ref instanceof ClickHouseTimeValue ? (ClickHouseTimeValue) ((ClickHouseTimeValue) ref).update(value) - : new ClickHouseTimeValue(value); - } - - protected ClickHouseTimeValue(LocalTime value) { - super(value); - } - - @Override - public ClickHouseTimeValue copy(boolean deep) { - return new ClickHouseTimeValue(getValue()); - } - - @Override - public byte asByte() { - return (byte) asInteger(); - } - - @Override - public short asShort() { - return (short) asInteger(); - } - - @Override - public int asInteger() { - return isNullOrEmpty() ? 0 : getValue().toSecondOfDay(); - } - - @Override - public long asLong() { - return asInteger(); - } - - @Override - public BigInteger asBigInteger() { - return isNullOrEmpty() ? null : BigInteger.valueOf(getValue().toSecondOfDay()); - } - - @Override - public float asFloat() { - return asInteger(); - } - - @Override - public double asDouble() { - return asInteger(); - } - - @Override - public BigDecimal asBigDecimal(int scale) { - return isNullOrEmpty() ? null : new BigDecimal(BigInteger.valueOf(getValue().toSecondOfDay()), scale); - } - - @Override - public LocalDate asDate() { - return isNullOrEmpty() ? null : ClickHouseValues.DATE_ZERO; - } - - @Override - public LocalTime asTime() { - return getValue(); - } - - @Override - public LocalDateTime asDateTime(int scale) { - return isNullOrEmpty() ? null : LocalDateTime.of(ClickHouseValues.DATE_ZERO, getValue()); - } - - @Override - public String asString(int length, Charset charset) { - if (isNullOrEmpty()) { - return null; - } - - String str = getValue().format(ClickHouseValues.TIME_FORMATTER); - if (length > 0) { - ClickHouseChecker.notWithDifferentLength(str.getBytes(charset == null ? StandardCharsets.UTF_8 : charset), - length); - } - - return str; - } - - @Override - public String toSqlExpression() { - if (isNullOrEmpty()) { - return ClickHouseValues.NULL_EXPR; - } - return new StringBuilder().append('\'').append(getValue().format(ClickHouseValues.TIME_FORMATTER)).append('\'') - .toString(); - } - - @Override - public ClickHouseTimeValue update(byte value) { - set(LocalTime.ofSecondOfDay(value)); - return this; - } - - @Override - public ClickHouseTimeValue update(short value) { - set(LocalTime.ofSecondOfDay(value)); - return this; - } - - @Override - public ClickHouseTimeValue update(int value) { - set(LocalTime.ofSecondOfDay(value)); - return this; - } - - @Override - public ClickHouseTimeValue update(long value) { - set(LocalTime.ofSecondOfDay(value)); - return this; - } - - @Override - public ClickHouseTimeValue update(float value) { - set(LocalTime.ofSecondOfDay((long) value)); - return this; - } - - @Override - public ClickHouseTimeValue update(double value) { - set(LocalTime.ofSecondOfDay((long) value)); - return this; - } - - @Override - public ClickHouseTimeValue update(BigInteger value) { - if (value == null) { - resetToNullOrEmpty(); - } else { - set(LocalTime.ofSecondOfDay(value.longValueExact())); - } - return this; - } - - @Override - public ClickHouseTimeValue update(BigDecimal value) { - if (value == null) { - resetToNullOrEmpty(); - } else { - set(LocalTime.ofSecondOfDay(value.longValueExact())); - } - return this; - } - - @Override - public ClickHouseTimeValue update(Enum value) { - if (value == null) { - resetToNullOrEmpty(); - } else { - set(LocalTime.ofSecondOfDay(value.ordinal())); - } - return this; - } - - @Override - public ClickHouseTimeValue update(LocalDate value) { - return this; - } - - @Override - public ClickHouseTimeValue update(LocalTime value) { - set(value); - return this; - } - - @Override - public ClickHouseTimeValue update(LocalDateTime value) { - if (value == null) { - resetToNullOrEmpty(); - } else { - set(value.toLocalTime()); - } - return this; - } - - @Override - public ClickHouseTimeValue update(String value) { - if (value == null) { - resetToNullOrEmpty(); - } else { - set(LocalTime.parse(value, ClickHouseValues.TIME_FORMATTER)); - } - return this; - } - - @Override - public ClickHouseTimeValue update(ClickHouseValue value) { - if (value == null) { - resetToNullOrEmpty(); - } else { - set(value.asTime()); - } - return this; - } - - @Override - public ClickHouseTimeValue update(Object value) { - if (value instanceof LocalTime) { - set((LocalTime) value); - } else if (value instanceof LocalDateTime) { - set(((LocalDateTime) value).toLocalTime()); - } else if (value instanceof LocalDate) { - set(LocalTime.MIN); - } else { - super.update(value); - } - return this; - } -} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTupleValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTupleValue.java index 1e176451b..a75f6a04b 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTupleValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTupleValue.java @@ -78,7 +78,7 @@ public ClickHouseTupleValue copy(boolean deep) { @Override public Object[] asArray() { if (isNullOrEmpty()) { - return ClickHouseValues.EMPTY_ARRAY; + return ClickHouseValues.EMPTY_OBJECT_ARRAY; } List value = getValue(); @@ -89,7 +89,7 @@ public Object[] asArray() { @SuppressWarnings("unchecked") public T[] asArray(Class clazz) { if (isNullOrEmpty()) { - return (T[]) ClickHouseValues.EMPTY_ARRAY; + return (T[]) ClickHouseValues.EMPTY_OBJECT_ARRAY; } List value = getValue(); @@ -334,48 +334,72 @@ public ClickHouseTupleValue update(double[] value) { @Override public ClickHouseTupleValue update(BigInteger value) { + if (value == null) { + return resetToNullOrEmpty(); + } set(Collections.singletonList(value)); return this; } @Override public ClickHouseTupleValue update(BigDecimal value) { + if (value == null) { + return resetToNullOrEmpty(); + } set(Collections.singletonList(value)); return this; } @Override public ClickHouseTupleValue update(Enum value) { + if (value == null) { + return resetToNullOrEmpty(); + } set(Collections.singletonList(value)); return this; } @Override public ClickHouseTupleValue update(Inet4Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } set(Collections.singletonList(value)); return this; } @Override public ClickHouseTupleValue update(Inet6Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } set(Collections.singletonList(value)); return this; } @Override public ClickHouseTupleValue update(LocalDate value) { + if (value == null) { + return resetToNullOrEmpty(); + } set(Collections.singletonList(value)); return this; } @Override public ClickHouseTupleValue update(LocalTime value) { + if (value == null) { + return resetToNullOrEmpty(); + } set(Collections.singletonList(value)); return this; } @Override public ClickHouseTupleValue update(LocalDateTime value) { + if (value == null) { + return resetToNullOrEmpty(); + } set(Collections.singletonList(value)); return this; } @@ -429,12 +453,18 @@ public ClickHouseTupleValue update(Map value) { @Override public ClickHouseTupleValue update(String value) { + if (value == null) { + return resetToNullOrEmpty(); + } set(Collections.singletonList(value)); return this; } @Override public ClickHouseTupleValue update(UUID value) { + if (value == null) { + return resetToNullOrEmpty(); + } set(Collections.singletonList(value)); return this; } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseByteArrayValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseByteArrayValue.java new file mode 100644 index 000000000..611185e1a --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseByteArrayValue.java @@ -0,0 +1,487 @@ +package com.clickhouse.client.data.array; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.Collection; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.Map.Entry; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.client.data.ClickHouseObjectValue; + +/** + * Wrapper of {@code byte[]}. + */ +public class ClickHouseByteArrayValue extends ClickHouseObjectValue { + private static final String TYPE_NAME = "byte[]"; + + /** + * Creates an empty array. + * + * @return empty array + */ + + public static ClickHouseByteArrayValue ofEmpty() { + return of(ClickHouseValues.EMPTY_BYTE_ARRAY); + } + + /** + * Wrap the given value. + * + * @param value value + * @return object representing the value + */ + public static ClickHouseByteArrayValue of(byte[] value) { + return of(null, value); + } + + /** + * Update value of the given object or create a new instance if {@code ref} is + * null. + * + * @param ref object to update, could be null + * @param value value + * @return same object as {@code ref} or a new instance if it's null + */ + + public static ClickHouseByteArrayValue of(ClickHouseValue ref, byte[] value) { + return ref instanceof ClickHouseByteArrayValue ? ((ClickHouseByteArrayValue) ref).set(value) + : new ClickHouseByteArrayValue(value); + } + + protected ClickHouseByteArrayValue(byte[] value) { + super(value); + } + + @Override + protected ClickHouseByteArrayValue set(byte[] value) { + super.set(ClickHouseChecker.nonNull(value, ClickHouseValues.TYPE_ARRAY)); + return this; + } + + @Override + public Object[] asArray() { + byte[] v = getValue(); + int len = v.length; + Byte[] array = new Byte[len]; + for (int i = 0; i < len; i++) { + array[i] = Byte.valueOf(v[i]); + } + return array; + } + + @Override + public E[] asArray(Class clazz) { + byte[] v = getValue(); + int len = v.length; + E[] array = ClickHouseValues.createObjectArray(clazz, len, 1); + for (int i = 0; i < len; i++) { + array[i] = clazz.cast(v[i]); + } + return array; + } + + @Override + public Map asMap(Class keyClass, Class valueClass) { + if (keyClass == null || valueClass == null) { + throw new IllegalArgumentException("Non-null key and value classes are required"); + } + byte[] v = getValue(); + Map map = new LinkedHashMap<>(); + for (int i = 0; i < v.length; i++) { + map.put(keyClass.cast(i + 1), valueClass.cast(v[i])); + } + // why not use Collections.unmodifiableMap(map) here? + return map; + } + + @Override + public String asString(int length, Charset charset) { + String str = Arrays.toString(getValue()); + if (length > 0) { + ClickHouseChecker.notWithDifferentLength(str.getBytes(charset == null ? StandardCharsets.UTF_8 : charset), + length); + } + + return str; + } + + @Override + public ClickHouseByteArrayValue copy(boolean deep) { + if (!deep) { + return new ClickHouseByteArrayValue(getValue()); + } + + byte[] value = getValue(); + return new ClickHouseByteArrayValue(Arrays.copyOf(value, value.length)); + } + + @Override + public boolean isNullOrEmpty() { + return getValue().length == 0; + } + + @Override + + public ClickHouseByteArrayValue resetToNullOrEmpty() { + set(ClickHouseValues.EMPTY_BYTE_ARRAY); + return this; + } + + @Override + public String toSqlExpression() { + byte[] value = getValue(); + int len = value == null ? 0 : value.length; + if (len == 0) { + return ClickHouseValues.EMPTY_ARRAY_EXPR; + } + + StringBuilder builder = new StringBuilder().append('['); + for (int i = 0; i < len; i++) { + builder.append(value[i]).append(','); + } + builder.setLength(builder.length() - 1); + return builder.append(']').toString(); + } + + @Override + public ClickHouseByteArrayValue update(boolean[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + byte[] v = new byte[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i] ? (byte) 1 : (byte) 0; + } + return set(v); + } + + @Override + public ClickHouseByteArrayValue update(char[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + byte[] v = new byte[len]; + for (int i = 0; i < len; i++) { + v[i] = (byte) value[i]; + } + return set(v); + } + + @Override + public ClickHouseByteArrayValue update(byte value) { + return set(new byte[] { value }); + } + + @Override + public ClickHouseByteArrayValue update(byte[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + return set(Arrays.copyOf(value, len)); + } + + @Override + public ClickHouseByteArrayValue update(short value) { + return set(new byte[] { (byte) value }); + } + + @Override + public ClickHouseByteArrayValue update(short[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + byte[] v = new byte[len]; + for (int i = 0; i < len; i++) { + v[i] = (byte) value[i]; + } + return set(v); + } + + @Override + public ClickHouseByteArrayValue update(int value) { + return set(new byte[] { (byte) value }); + } + + @Override + public ClickHouseByteArrayValue update(int[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + byte[] v = new byte[len]; + for (int i = 0; i < len; i++) { + v[i] = (byte) value[i]; + } + return set(v); + } + + @Override + public ClickHouseByteArrayValue update(long value) { + return set(new byte[] { (byte) value }); + } + + @Override + public ClickHouseByteArrayValue update(long[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + byte[] v = new byte[len]; + for (int i = 0; i < len; i++) { + v[i] = (byte) value[i]; + } + return set(v); + } + + @Override + public ClickHouseByteArrayValue update(float value) { + return set(new byte[] { (byte) value }); + } + + @Override + public ClickHouseByteArrayValue update(float[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + byte[] v = new byte[len]; + for (int i = 0; i < len; i++) { + v[i] = (byte) value[i]; + } + return set(v); + } + + @Override + public ClickHouseByteArrayValue update(double value) { + return set(new byte[] { (byte) value }); + } + + @Override + public ClickHouseByteArrayValue update(double[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + byte[] v = new byte[len]; + for (int i = 0; i < len; i++) { + v[i] = (byte) value[i]; + } + return set(v); + } + + @Override + public ClickHouseByteArrayValue update(BigInteger value) { + return set(value == null ? ClickHouseValues.EMPTY_BYTE_ARRAY : new byte[] { value.byteValue() }); + } + + @Override + public ClickHouseByteArrayValue update(BigDecimal value) { + return set(value == null ? ClickHouseValues.EMPTY_BYTE_ARRAY : new byte[] { value.byteValue() }); + } + + @Override + public ClickHouseByteArrayValue update(Enum value) { + return set(value == null ? ClickHouseValues.EMPTY_BYTE_ARRAY : new byte[] { (byte) value.ordinal() }); + } + + @Override + public ClickHouseByteArrayValue update(Inet4Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + // return set(value == null ? ClickHouseValues.EMPTY_BYTE_ARRAY : + // value.getAddress()); + throw newUnsupportedException(ClickHouseValues.TYPE_IPV4, TYPE_NAME); + } + + @Override + public ClickHouseByteArrayValue update(Inet6Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + // return set(value == null ? ClickHouseValues.EMPTY_BYTE_ARRAY : + // value.getAddress()); + throw newUnsupportedException(ClickHouseValues.TYPE_IPV6, TYPE_NAME); + } + + @Override + public ClickHouseByteArrayValue update(LocalDate value) { + return set(value == null ? ClickHouseValues.EMPTY_BYTE_ARRAY : new byte[] { (byte) value.toEpochDay() }); + } + + @Override + public ClickHouseByteArrayValue update(LocalTime value) { + return set(value == null ? ClickHouseValues.EMPTY_BYTE_ARRAY : new byte[] { (byte) value.toSecondOfDay() }); + } + + @Override + public ClickHouseByteArrayValue update(LocalDateTime value) { + return set(value == null ? ClickHouseValues.EMPTY_BYTE_ARRAY + : new byte[] { (byte) value.toEpochSecond(ZoneOffset.UTC) }); + } + + @Override + public ClickHouseByteArrayValue update(Collection value) { + int size = value == null ? 0 : value.size(); + if (size == 0) { + return resetToNullOrEmpty(); + } + + byte[] v = new byte[size]; + int index = 0; + for (Object o : value) { + v[index++] = o == null ? 0 : ((Number) o).byteValue(); + } + return set(v); + } + + @Override + public ClickHouseByteArrayValue update(Enumeration value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + List v = new LinkedList<>(); + while (value.hasMoreElements()) { + v.add((Number) value.nextElement()); + } + + byte[] values = new byte[v.size()]; + int index = 0; + for (Number n : v) { + values[index++] = n == null ? 0 : n.byteValue(); + } + return set(values); + } + + @Override + public ClickHouseByteArrayValue update(Map value) { + int size = value == null ? 0 : value.size(); + if (size == 0) { + return resetToNullOrEmpty(); + } + + byte[] v = new byte[size]; + int index = 0; + for (Entry e : value.entrySet()) { + Object o = e.getValue(); + v[index++] = o == null ? 0 : ((Number) e.getValue()).byteValue(); + } + return set(v); + } + + @Override + public ClickHouseByteArrayValue update(String value) { + return set(new byte[] { Byte.parseByte(value) }); + } + + @Override + public ClickHouseByteArrayValue update(UUID value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + // ByteBuffer buffer = ByteBuffer.wrap(new byte[16]); + // buffer.putLong(value.getMostSignificantBits()); + // buffer.putLong(value.getLeastSignificantBits()); + // return set(buffer.array()); + throw newUnsupportedException(ClickHouseValues.TYPE_UUID, TYPE_NAME); + } + + @Override + public ClickHouseByteArrayValue update(ClickHouseValue value) { + if (value == null || value.isNullOrEmpty()) { + return resetToNullOrEmpty(); + } else if (value instanceof ClickHouseByteArrayValue) { + set(((ClickHouseByteArrayValue) value).getValue()); + } else { + update(value.asArray()); + } + return this; + } + + @Override + public ClickHouseByteArrayValue update(Object[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } else { + byte[] values = new byte[len]; + for (int i = 0; i < len; i++) { + Object o = value[i]; + values[i] = o == null ? 0 : ((Number) o).byteValue(); + } + set(values); + } + + return this; + } + + @Override + public ClickHouseValue updateUnknown(Object value) { + if (value == null) { + return resetToNullOrEmpty(); + } else if (value instanceof Number) { + return set(new byte[] { ((Number) value).byteValue() }); + } else { + throw newUnsupportedException(value.getClass().getName(), TYPE_NAME); + } + } + + @Override + public ClickHouseByteArrayValue update(Object value) { + if (value instanceof byte[]) { + set((byte[]) value); + } else { + super.update(value); + } + return this; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { // too bad this is a mutable class :< + return true; + } else if (obj == null || getClass() != obj.getClass()) { + return false; + } + + return Arrays.equals(getValue(), ((ClickHouseByteArrayValue) obj).getValue()); + } + + @Override + public int hashCode() { + return Arrays.hashCode(getValue()); + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseDoubleArrayValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseDoubleArrayValue.java new file mode 100644 index 000000000..cbba6fd80 --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseDoubleArrayValue.java @@ -0,0 +1,479 @@ +package com.clickhouse.client.data.array; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.Collection; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.Map.Entry; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.client.data.ClickHouseObjectValue; + +/** + * Wrapper of {@code double[]}. + */ +public class ClickHouseDoubleArrayValue extends ClickHouseObjectValue { + private static final String TYPE_NAME = "double[]"; + + /** + * Creates an empty array. + * + * @return empty array + */ + + public static ClickHouseDoubleArrayValue ofEmpty() { + return of(ClickHouseValues.EMPTY_DOUBLE_ARRAY); + } + + /** + * Wrap the given value. + * + * @param value value + * @return object representing the value + */ + public static ClickHouseDoubleArrayValue of(double[] value) { + return of(null, value); + } + + /** + * Update value of the given object or create a new instance if {@code ref} is + * null. + * + * @param ref object to update, could be null + * @param value value + * @return same object as {@code ref} or a new instance if it's null + */ + + public static ClickHouseDoubleArrayValue of(ClickHouseValue ref, double[] value) { + return ref instanceof ClickHouseDoubleArrayValue ? ((ClickHouseDoubleArrayValue) ref).set(value) + : new ClickHouseDoubleArrayValue(value); + } + + protected ClickHouseDoubleArrayValue(double[] value) { + super(value); + } + + @Override + protected ClickHouseDoubleArrayValue set(double[] value) { + super.set(ClickHouseChecker.nonNull(value, ClickHouseValues.TYPE_ARRAY)); + return this; + } + + @Override + public Object[] asArray() { + double[] v = getValue(); + int len = v.length; + Double[] array = new Double[len]; + for (int i = 0; i < len; i++) { + array[i] = Double.valueOf(v[i]); + } + return array; + } + + @Override + public E[] asArray(Class clazz) { + double[] v = getValue(); + int len = v.length; + E[] array = ClickHouseValues.createObjectArray(clazz, len, 1); + for (int i = 0; i < len; i++) { + array[i] = clazz.cast(v[i]); + } + return array; + } + + @Override + public Map asMap(Class keyClass, Class valueClass) { + if (keyClass == null || valueClass == null) { + throw new IllegalArgumentException("Non-null key and value classes are required"); + } + double[] v = getValue(); + Map map = new LinkedHashMap<>(); + for (int i = 0; i < v.length; i++) { + map.put(keyClass.cast(i + 1), valueClass.cast(v[i])); + } + // why not use Collections.unmodifiableMap(map) here? + return map; + } + + @Override + public String asString(int length, Charset charset) { + String str = Arrays.toString(getValue()); + if (length > 0) { + ClickHouseChecker.notWithDifferentLength(str.getBytes(charset == null ? StandardCharsets.UTF_8 : charset), + length); + } + + return str; + } + + @Override + public ClickHouseDoubleArrayValue copy(boolean deep) { + if (!deep) { + return new ClickHouseDoubleArrayValue(getValue()); + } + + double[] value = getValue(); + return new ClickHouseDoubleArrayValue(Arrays.copyOf(value, value.length)); + } + + @Override + public boolean isNullOrEmpty() { + return getValue().length == 0; + } + + @Override + + public ClickHouseDoubleArrayValue resetToNullOrEmpty() { + set(ClickHouseValues.EMPTY_DOUBLE_ARRAY); + return this; + } + + @Override + public String toSqlExpression() { + double[] value = getValue(); + int len = value == null ? 0 : value.length; + if (len == 0) { + return ClickHouseValues.EMPTY_ARRAY_EXPR; + } + + StringBuilder builder = new StringBuilder().append('['); + for (int i = 0; i < len; i++) { + builder.append(value[i]).append(','); + } + builder.setLength(builder.length() - 1); + return builder.append(']').toString(); + } + + @Override + public ClickHouseDoubleArrayValue update(boolean[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + double[] v = new double[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i] ? 1D : 0D; + } + return set(v); + } + + @Override + public ClickHouseDoubleArrayValue update(char[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + double[] v = new double[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseDoubleArrayValue update(byte value) { + return set(new double[] { value }); + } + + @Override + public ClickHouseDoubleArrayValue update(byte[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + double[] v = new double[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseDoubleArrayValue update(short value) { + return set(new double[] { value }); + } + + @Override + public ClickHouseDoubleArrayValue update(short[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + double[] v = new double[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseDoubleArrayValue update(int value) { + return set(new double[] { value }); + } + + @Override + public ClickHouseDoubleArrayValue update(int[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + double[] v = new double[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseDoubleArrayValue update(long value) { + return set(new double[] { value }); + } + + @Override + public ClickHouseDoubleArrayValue update(long[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + double[] v = new double[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseDoubleArrayValue update(float value) { + return set(new double[] { value }); + } + + @Override + public ClickHouseDoubleArrayValue update(float[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + double[] v = new double[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseDoubleArrayValue update(double value) { + return set(new double[] { value }); + } + + @Override + public ClickHouseDoubleArrayValue update(double[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + return set(Arrays.copyOf(value, len)); + } + + @Override + public ClickHouseDoubleArrayValue update(BigInteger value) { + return set(value == null ? ClickHouseValues.EMPTY_DOUBLE_ARRAY : new double[] { value.doubleValue() }); + } + + @Override + public ClickHouseDoubleArrayValue update(BigDecimal value) { + return set(value == null ? ClickHouseValues.EMPTY_DOUBLE_ARRAY : new double[] { value.doubleValue() }); + } + + @Override + public ClickHouseDoubleArrayValue update(Enum value) { + return set(value == null ? ClickHouseValues.EMPTY_DOUBLE_ARRAY : new double[] { value.ordinal() }); + } + + @Override + public ClickHouseDoubleArrayValue update(Inet4Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_IPV4, TYPE_NAME); + } + + @Override + public ClickHouseDoubleArrayValue update(Inet6Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_IPV6, TYPE_NAME); + } + + @Override + public ClickHouseDoubleArrayValue update(LocalDate value) { + return set(value == null ? ClickHouseValues.EMPTY_DOUBLE_ARRAY : new double[] { value.toEpochDay() }); + } + + @Override + public ClickHouseDoubleArrayValue update(LocalTime value) { + return set(value == null ? ClickHouseValues.EMPTY_DOUBLE_ARRAY : new double[] { value.toSecondOfDay() }); + } + + @Override + public ClickHouseDoubleArrayValue update(LocalDateTime value) { + return set(value == null ? ClickHouseValues.EMPTY_DOUBLE_ARRAY + : new double[] { value.toEpochSecond(ZoneOffset.UTC) }); + } + + @Override + public ClickHouseDoubleArrayValue update(Collection value) { + int size = value == null ? 0 : value.size(); + if (size == 0) { + return resetToNullOrEmpty(); + } + + double[] v = new double[size]; + int index = 0; + for (Object o : value) { + v[index++] = o == null ? 0 : ((Number) o).doubleValue(); + } + return set(v); + } + + @Override + public ClickHouseDoubleArrayValue update(Enumeration value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + List v = new LinkedList<>(); + while (value.hasMoreElements()) { + v.add((Number) value.nextElement()); + } + + double[] values = new double[v.size()]; + int index = 0; + for (Number n : v) { + values[index++] = n == null ? 0 : n.doubleValue(); + } + return set(values); + } + + @Override + public ClickHouseDoubleArrayValue update(Map value) { + int size = value == null ? 0 : value.size(); + if (size == 0) { + return resetToNullOrEmpty(); + } + + double[] v = new double[size]; + int index = 0; + for (Entry e : value.entrySet()) { + Object o = e.getValue(); + v[index++] = o == null ? 0 : ((Number) e.getValue()).doubleValue(); + } + return set(v); + } + + @Override + public ClickHouseDoubleArrayValue update(String value) { + return set(new double[] { Double.parseDouble(value) }); + } + + @Override + public ClickHouseDoubleArrayValue update(UUID value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_UUID, TYPE_NAME); + } + + @Override + public ClickHouseDoubleArrayValue update(ClickHouseValue value) { + if (value == null || value.isNullOrEmpty()) { + return resetToNullOrEmpty(); + } else if (value instanceof ClickHouseDoubleArrayValue) { + set(((ClickHouseDoubleArrayValue) value).getValue()); + } else { + update(value.asArray()); + } + return this; + } + + @Override + public ClickHouseDoubleArrayValue update(Object[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } else { + double[] values = new double[len]; + for (int i = 0; i < len; i++) { + Object o = value[i]; + values[i] = o == null ? 0 : ((Number) o).doubleValue(); + } + set(values); + } + + return this; + } + + @Override + public ClickHouseValue updateUnknown(Object value) { + if (value == null) { + return resetToNullOrEmpty(); + } else if (value instanceof Number) { + return set(new double[] { ((Number) value).doubleValue() }); + } else { + throw newUnsupportedException(value.getClass().getName(), TYPE_NAME); + } + } + + @Override + public ClickHouseDoubleArrayValue update(Object value) { + if (value instanceof double[]) { + set((double[]) value); + } else { + super.update(value); + } + return this; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { // too bad this is a mutable class :< + return true; + } else if (obj == null || getClass() != obj.getClass()) { + return false; + } + + return Arrays.equals(getValue(), ((ClickHouseDoubleArrayValue) obj).getValue()); + } + + @Override + public int hashCode() { + return Arrays.hashCode(getValue()); + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseFloatArrayValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseFloatArrayValue.java new file mode 100644 index 000000000..b608df867 --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseFloatArrayValue.java @@ -0,0 +1,479 @@ +package com.clickhouse.client.data.array; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.Collection; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.Map.Entry; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.client.data.ClickHouseObjectValue; + +/** + * Wrapper of {@code float[]}. + */ +public class ClickHouseFloatArrayValue extends ClickHouseObjectValue { + private static final String TYPE_NAME = "float[]"; + + /** + * Creates an empty array. + * + * @return empty array + */ + + public static ClickHouseFloatArrayValue ofEmpty() { + return of(ClickHouseValues.EMPTY_FLOAT_ARRAY); + } + + /** + * Wrap the given value. + * + * @param value value + * @return object representing the value + */ + public static ClickHouseFloatArrayValue of(float[] value) { + return of(null, value); + } + + /** + * Update value of the given object or create a new instance if {@code ref} is + * null. + * + * @param ref object to update, could be null + * @param value value + * @return same object as {@code ref} or a new instance if it's null + */ + + public static ClickHouseFloatArrayValue of(ClickHouseValue ref, float[] value) { + return ref instanceof ClickHouseFloatArrayValue ? ((ClickHouseFloatArrayValue) ref).set(value) + : new ClickHouseFloatArrayValue(value); + } + + protected ClickHouseFloatArrayValue(float[] value) { + super(value); + } + + @Override + protected ClickHouseFloatArrayValue set(float[] value) { + super.set(ClickHouseChecker.nonNull(value, ClickHouseValues.TYPE_ARRAY)); + return this; + } + + @Override + public Object[] asArray() { + float[] v = getValue(); + int len = v.length; + Float[] array = new Float[len]; + for (int i = 0; i < len; i++) { + array[i] = Float.valueOf(v[i]); + } + return array; + } + + @Override + public E[] asArray(Class clazz) { + float[] v = getValue(); + int len = v.length; + E[] array = ClickHouseValues.createObjectArray(clazz, len, 1); + for (int i = 0; i < len; i++) { + array[i] = clazz.cast(v[i]); + } + return array; + } + + @Override + public Map asMap(Class keyClass, Class valueClass) { + if (keyClass == null || valueClass == null) { + throw new IllegalArgumentException("Non-null key and value classes are required"); + } + float[] v = getValue(); + Map map = new LinkedHashMap<>(); + for (int i = 0; i < v.length; i++) { + map.put(keyClass.cast(i + 1), valueClass.cast(v[i])); + } + // why not use Collections.unmodifiableMap(map) here? + return map; + } + + @Override + public String asString(int length, Charset charset) { + String str = Arrays.toString(getValue()); + if (length > 0) { + ClickHouseChecker.notWithDifferentLength(str.getBytes(charset == null ? StandardCharsets.UTF_8 : charset), + length); + } + + return str; + } + + @Override + public ClickHouseFloatArrayValue copy(boolean deep) { + if (!deep) { + return new ClickHouseFloatArrayValue(getValue()); + } + + float[] value = getValue(); + return new ClickHouseFloatArrayValue(Arrays.copyOf(value, value.length)); + } + + @Override + public boolean isNullOrEmpty() { + return getValue().length == 0; + } + + @Override + + public ClickHouseFloatArrayValue resetToNullOrEmpty() { + set(ClickHouseValues.EMPTY_FLOAT_ARRAY); + return this; + } + + @Override + public String toSqlExpression() { + float[] value = getValue(); + int len = value == null ? 0 : value.length; + if (len == 0) { + return ClickHouseValues.EMPTY_ARRAY_EXPR; + } + + StringBuilder builder = new StringBuilder().append('['); + for (int i = 0; i < len; i++) { + builder.append(value[i]).append(','); + } + builder.setLength(builder.length() - 1); + return builder.append(']').toString(); + } + + @Override + public ClickHouseFloatArrayValue update(boolean[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + float[] v = new float[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i] ? 1F : (float) 0; + } + return set(v); + } + + @Override + public ClickHouseFloatArrayValue update(char[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + float[] v = new float[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseFloatArrayValue update(byte value) { + return set(new float[] { value }); + } + + @Override + public ClickHouseFloatArrayValue update(byte[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + float[] v = new float[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseFloatArrayValue update(short value) { + return set(new float[] { value }); + } + + @Override + public ClickHouseFloatArrayValue update(short[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + float[] v = new float[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseFloatArrayValue update(int value) { + return set(new float[] { value }); + } + + @Override + public ClickHouseFloatArrayValue update(int[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + float[] v = new float[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseFloatArrayValue update(long value) { + return set(new float[] { value }); + } + + @Override + public ClickHouseFloatArrayValue update(long[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + float[] v = new float[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseFloatArrayValue update(float value) { + return set(new float[] { value }); + } + + @Override + public ClickHouseFloatArrayValue update(float[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + return set(Arrays.copyOf(value, len)); + } + + @Override + public ClickHouseFloatArrayValue update(double value) { + return set(new float[] { (float) value }); + } + + @Override + public ClickHouseFloatArrayValue update(double[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + float[] v = new float[len]; + for (int i = 0; i < len; i++) { + v[i] = (float) value[i]; + } + return set(v); + } + + @Override + public ClickHouseFloatArrayValue update(BigInteger value) { + return set(value == null ? ClickHouseValues.EMPTY_FLOAT_ARRAY : new float[] { value.floatValue() }); + } + + @Override + public ClickHouseFloatArrayValue update(BigDecimal value) { + return set(value == null ? ClickHouseValues.EMPTY_FLOAT_ARRAY : new float[] { value.floatValue() }); + } + + @Override + public ClickHouseFloatArrayValue update(Enum value) { + return set(value == null ? ClickHouseValues.EMPTY_FLOAT_ARRAY : new float[] { value.ordinal() }); + } + + @Override + public ClickHouseFloatArrayValue update(Inet4Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_IPV4, TYPE_NAME); + } + + @Override + public ClickHouseFloatArrayValue update(Inet6Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_IPV6, TYPE_NAME); + } + + @Override + public ClickHouseFloatArrayValue update(LocalDate value) { + return set(value == null ? ClickHouseValues.EMPTY_FLOAT_ARRAY : new float[] { value.toEpochDay() }); + } + + @Override + public ClickHouseFloatArrayValue update(LocalTime value) { + return set(value == null ? ClickHouseValues.EMPTY_FLOAT_ARRAY : new float[] { value.toSecondOfDay() }); + } + + @Override + public ClickHouseFloatArrayValue update(LocalDateTime value) { + return set(value == null ? ClickHouseValues.EMPTY_FLOAT_ARRAY + : new float[] { value.toEpochSecond(ZoneOffset.UTC) }); + } + + @Override + public ClickHouseFloatArrayValue update(Collection value) { + int size = value == null ? 0 : value.size(); + if (size == 0) { + return resetToNullOrEmpty(); + } + + float[] v = new float[size]; + int index = 0; + for (Object o : value) { + v[index++] = o == null ? 0 : ((Number) o).floatValue(); + } + return set(v); + } + + @Override + public ClickHouseFloatArrayValue update(Enumeration value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + List v = new LinkedList<>(); + while (value.hasMoreElements()) { + v.add((Number) value.nextElement()); + } + + float[] values = new float[v.size()]; + int index = 0; + for (Number n : v) { + values[index++] = n == null ? 0 : n.floatValue(); + } + return set(values); + } + + @Override + public ClickHouseFloatArrayValue update(Map value) { + int size = value == null ? 0 : value.size(); + if (size == 0) { + return resetToNullOrEmpty(); + } + + float[] v = new float[size]; + int index = 0; + for (Entry e : value.entrySet()) { + Object o = e.getValue(); + v[index++] = o == null ? 0 : ((Number) e.getValue()).floatValue(); + } + return set(v); + } + + @Override + public ClickHouseFloatArrayValue update(String value) { + return set(new float[] { Float.parseFloat(value) }); + } + + @Override + public ClickHouseFloatArrayValue update(UUID value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_UUID, TYPE_NAME); + } + + @Override + public ClickHouseFloatArrayValue update(ClickHouseValue value) { + if (value == null || value.isNullOrEmpty()) { + return resetToNullOrEmpty(); + } else if (value instanceof ClickHouseFloatArrayValue) { + set(((ClickHouseFloatArrayValue) value).getValue()); + } else { + update(value.asArray()); + } + return this; + } + + @Override + public ClickHouseFloatArrayValue update(Object[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } else { + float[] values = new float[len]; + for (int i = 0; i < len; i++) { + Object o = value[i]; + values[i] = o == null ? 0 : ((Number) o).floatValue(); + } + set(values); + } + + return this; + } + + @Override + public ClickHouseValue updateUnknown(Object value) { + if (value == null) { + return resetToNullOrEmpty(); + } else if (value instanceof Number) { + return set(new float[] { ((Number) value).floatValue() }); + } else { + throw newUnsupportedException(value.getClass().getName(), TYPE_NAME); + } + } + + @Override + public ClickHouseFloatArrayValue update(Object value) { + if (value instanceof float[]) { + set((float[]) value); + } else { + super.update(value); + } + return this; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { // too bad this is a mutable class :< + return true; + } else if (obj == null || getClass() != obj.getClass()) { + return false; + } + + return Arrays.equals(getValue(), ((ClickHouseFloatArrayValue) obj).getValue()); + } + + @Override + public int hashCode() { + return Arrays.hashCode(getValue()); + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseIntArrayValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseIntArrayValue.java new file mode 100644 index 000000000..955bc42ea --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseIntArrayValue.java @@ -0,0 +1,479 @@ +package com.clickhouse.client.data.array; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.Collection; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.Map.Entry; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.client.data.ClickHouseObjectValue; + +/** + * Wrapper of {@code int[]}. + */ +public class ClickHouseIntArrayValue extends ClickHouseObjectValue { + private final static String TYPE_NAME = "int[]"; + + /** + * Creates an empty array. + * + * @return empty array + */ + + public static ClickHouseIntArrayValue ofEmpty() { + return of(ClickHouseValues.EMPTY_INT_ARRAY); + } + + /** + * Wrap the given value. + * + * @param value value + * @return object representing the value + */ + public static ClickHouseIntArrayValue of(int[] value) { + return of(null, value); + } + + /** + * Update value of the given object or create a new instance if {@code ref} is + * null. + * + * @param ref object to update, could be null + * @param value value + * @return same object as {@code ref} or a new instance if it's null + */ + + public static ClickHouseIntArrayValue of(ClickHouseValue ref, int[] value) { + return ref instanceof ClickHouseIntArrayValue ? ((ClickHouseIntArrayValue) ref).set(value) + : new ClickHouseIntArrayValue(value); + } + + protected ClickHouseIntArrayValue(int[] value) { + super(value); + } + + @Override + protected ClickHouseIntArrayValue set(int[] value) { + super.set(ClickHouseChecker.nonNull(value, ClickHouseValues.TYPE_ARRAY)); + return this; + } + + @Override + public Object[] asArray() { + int[] v = getValue(); + int len = v.length; + Integer[] array = new Integer[len]; + for (int i = 0; i < len; i++) { + array[i] = Integer.valueOf(v[i]); + } + return array; + } + + @Override + public E[] asArray(Class clazz) { + int[] v = getValue(); + int len = v.length; + E[] array = ClickHouseValues.createObjectArray(clazz, len, 1); + for (int i = 0; i < len; i++) { + array[i] = clazz.cast(v[i]); + } + return array; + } + + @Override + public Map asMap(Class keyClass, Class valueClass) { + if (keyClass == null || valueClass == null) { + throw new IllegalArgumentException("Non-null key and value classes are required"); + } + int[] v = getValue(); + Map map = new LinkedHashMap<>(); + for (int i = 0; i < v.length; i++) { + map.put(keyClass.cast(i + 1), valueClass.cast(v[i])); + } + // why not use Collections.unmodifiableMap(map) here? + return map; + } + + @Override + public String asString(int length, Charset charset) { + String str = Arrays.toString(getValue()); + if (length > 0) { + ClickHouseChecker.notWithDifferentLength(str.getBytes(charset == null ? StandardCharsets.UTF_8 : charset), + length); + } + + return str; + } + + @Override + public ClickHouseIntArrayValue copy(boolean deep) { + if (!deep) { + return new ClickHouseIntArrayValue(getValue()); + } + + int[] value = getValue(); + return new ClickHouseIntArrayValue(Arrays.copyOf(value, value.length)); + } + + @Override + public boolean isNullOrEmpty() { + return getValue().length == 0; + } + + @Override + + public ClickHouseIntArrayValue resetToNullOrEmpty() { + set(ClickHouseValues.EMPTY_INT_ARRAY); + return this; + } + + @Override + public String toSqlExpression() { + int[] value = getValue(); + int len = value == null ? 0 : value.length; + if (len == 0) { + return ClickHouseValues.EMPTY_ARRAY_EXPR; + } + + StringBuilder builder = new StringBuilder().append('['); + for (int i = 0; i < len; i++) { + builder.append(value[i]).append(','); + } + builder.setLength(builder.length() - 1); + return builder.append(']').toString(); + } + + @Override + public ClickHouseIntArrayValue update(boolean[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + int[] v = new int[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i] ? 1 : 0; + } + return set(v); + } + + @Override + public ClickHouseIntArrayValue update(char[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + int[] v = new int[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseIntArrayValue update(byte value) { + return set(new int[] { value }); + } + + @Override + public ClickHouseIntArrayValue update(byte[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + int[] v = new int[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseIntArrayValue update(short value) { + return set(new int[] { value }); + } + + @Override + public ClickHouseIntArrayValue update(short[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + int[] v = new int[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseIntArrayValue update(int value) { + return set(new int[] { value }); + } + + @Override + public ClickHouseIntArrayValue update(int[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + return set(Arrays.copyOf(value, len)); + } + + @Override + public ClickHouseIntArrayValue update(long value) { + return set(new int[] { (int) value }); + } + + @Override + public ClickHouseIntArrayValue update(long[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + int[] v = new int[len]; + for (int i = 0; i < len; i++) { + v[i] = (int) value[i]; + } + return set(v); + } + + @Override + public ClickHouseIntArrayValue update(float value) { + return set(new int[] { (int) value }); + } + + @Override + public ClickHouseIntArrayValue update(float[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + int[] v = new int[len]; + for (int i = 0; i < len; i++) { + v[i] = (int) value[i]; + } + return set(v); + } + + @Override + public ClickHouseIntArrayValue update(double value) { + return set(new int[] { (int) value }); + } + + @Override + public ClickHouseIntArrayValue update(double[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + int[] v = new int[len]; + for (int i = 0; i < len; i++) { + v[i] = (int) value[i]; + } + return set(v); + } + + @Override + public ClickHouseIntArrayValue update(BigInteger value) { + return set(value == null ? ClickHouseValues.EMPTY_INT_ARRAY : new int[] { value.intValue() }); + } + + @Override + public ClickHouseIntArrayValue update(BigDecimal value) { + return set(value == null ? ClickHouseValues.EMPTY_INT_ARRAY : new int[] { value.intValue() }); + } + + @Override + public ClickHouseIntArrayValue update(Enum value) { + return set(value == null ? ClickHouseValues.EMPTY_INT_ARRAY : new int[] { value.ordinal() }); + } + + @Override + public ClickHouseIntArrayValue update(Inet4Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_IPV4, TYPE_NAME); + } + + @Override + public ClickHouseIntArrayValue update(Inet6Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_IPV6, TYPE_NAME); + } + + @Override + public ClickHouseIntArrayValue update(LocalDate value) { + return set(value == null ? ClickHouseValues.EMPTY_INT_ARRAY : new int[] { (int) value.toEpochDay() }); + } + + @Override + public ClickHouseIntArrayValue update(LocalTime value) { + return set(value == null ? ClickHouseValues.EMPTY_INT_ARRAY : new int[] { value.toSecondOfDay() }); + } + + @Override + public ClickHouseIntArrayValue update(LocalDateTime value) { + return set(value == null ? ClickHouseValues.EMPTY_INT_ARRAY + : new int[] { (int) value.toEpochSecond(ZoneOffset.UTC) }); + } + + @Override + public ClickHouseIntArrayValue update(Collection value) { + int size = value == null ? 0 : value.size(); + if (size == 0) { + return resetToNullOrEmpty(); + } + + int[] v = new int[size]; + int index = 0; + for (Object o : value) { + v[index++] = o == null ? 0 : ((Number) o).intValue(); + } + return set(v); + } + + @Override + public ClickHouseIntArrayValue update(Enumeration value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + List v = new LinkedList<>(); + while (value.hasMoreElements()) { + v.add((Number) value.nextElement()); + } + + int[] values = new int[v.size()]; + int index = 0; + for (Number n : v) { + values[index++] = n == null ? 0 : n.intValue(); + } + return set(values); + } + + @Override + public ClickHouseIntArrayValue update(Map value) { + int size = value == null ? 0 : value.size(); + if (size == 0) { + return resetToNullOrEmpty(); + } + + int[] v = new int[size]; + int index = 0; + for (Entry e : value.entrySet()) { + Object o = e.getValue(); + v[index++] = o == null ? 0 : ((Number) e.getValue()).intValue(); + } + return set(v); + } + + @Override + public ClickHouseIntArrayValue update(String value) { + return set(new int[] { Integer.parseInt(value) }); + } + + @Override + public ClickHouseIntArrayValue update(UUID value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_UUID, TYPE_NAME); + } + + @Override + public ClickHouseIntArrayValue update(ClickHouseValue value) { + if (value == null || value.isNullOrEmpty()) { + return resetToNullOrEmpty(); + } else if (value instanceof ClickHouseIntArrayValue) { + set(((ClickHouseIntArrayValue) value).getValue()); + } else { + update(value.asArray()); + } + return this; + } + + @Override + public ClickHouseIntArrayValue update(Object[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } else { + int[] values = new int[len]; + for (int i = 0; i < len; i++) { + Object o = value[i]; + values[i] = o == null ? 0 : ((Number) o).intValue(); + } + set(values); + } + + return this; + } + + @Override + public ClickHouseValue updateUnknown(Object value) { + if (value == null) { + return resetToNullOrEmpty(); + } else if (value instanceof Number) { + return set(new int[] { ((Number) value).intValue() }); + } else { + throw newUnsupportedException(value.getClass().getName(), TYPE_NAME); + } + } + + @Override + public ClickHouseIntArrayValue update(Object value) { + if (value instanceof int[]) { + set((int[]) value); + } else { + super.update(value); + } + return this; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { // too bad this is a mutable class :< + return true; + } else if (obj == null || getClass() != obj.getClass()) { + return false; + } + + return Arrays.equals(getValue(), ((ClickHouseIntArrayValue) obj).getValue()); + } + + @Override + public int hashCode() { + return Arrays.hashCode(getValue()); + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseLongArrayValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseLongArrayValue.java new file mode 100644 index 000000000..6423b25f1 --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseLongArrayValue.java @@ -0,0 +1,479 @@ +package com.clickhouse.client.data.array; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.Collection; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.Map.Entry; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.client.data.ClickHouseObjectValue; + +/** + * Wrapper of {@code long[]}. + */ +public class ClickHouseLongArrayValue extends ClickHouseObjectValue { + private static final String TYPE_NAME = "long[]"; + + /** + * Creates an empty array. + * + * @return empty array + */ + + public static ClickHouseLongArrayValue ofEmpty() { + return of(ClickHouseValues.EMPTY_LONG_ARRAY); + } + + /** + * Wrap the given value. + * + * @param value value + * @return object representing the value + */ + public static ClickHouseLongArrayValue of(long[] value) { + return of(null, value); + } + + /** + * Update value of the given object or create a new instance if {@code ref} is + * null. + * + * @param ref object to update, could be null + * @param value value + * @return same object as {@code ref} or a new instance if it's null + */ + + public static ClickHouseLongArrayValue of(ClickHouseValue ref, long[] value) { + return ref instanceof ClickHouseLongArrayValue ? ((ClickHouseLongArrayValue) ref).set(value) + : new ClickHouseLongArrayValue(value); + } + + protected ClickHouseLongArrayValue(long[] value) { + super(value); + } + + @Override + protected ClickHouseLongArrayValue set(long[] value) { + super.set(ClickHouseChecker.nonNull(value, ClickHouseValues.TYPE_ARRAY)); + return this; + } + + @Override + public Object[] asArray() { + long[] v = getValue(); + int len = v.length; + Long[] array = new Long[len]; + for (int i = 0; i < len; i++) { + array[i] = Long.valueOf(v[i]); + } + return array; + } + + @Override + public E[] asArray(Class clazz) { + long[] v = getValue(); + int len = v.length; + E[] array = ClickHouseValues.createObjectArray(clazz, len, 1); + for (int i = 0; i < len; i++) { + array[i] = clazz.cast(v[i]); + } + return array; + } + + @Override + public Map asMap(Class keyClass, Class valueClass) { + if (keyClass == null || valueClass == null) { + throw new IllegalArgumentException("Non-null key and value classes are required"); + } + long[] v = getValue(); + Map map = new LinkedHashMap<>(); + for (int i = 0; i < v.length; i++) { + map.put(keyClass.cast(i + 1), valueClass.cast(v[i])); + } + // why not use Collections.unmodifiableMap(map) here? + return map; + } + + @Override + public String asString(int length, Charset charset) { + String str = Arrays.toString(getValue()); + if (length > 0) { + ClickHouseChecker.notWithDifferentLength(str.getBytes(charset == null ? StandardCharsets.UTF_8 : charset), + length); + } + + return str; + } + + @Override + public ClickHouseLongArrayValue copy(boolean deep) { + if (!deep) { + return new ClickHouseLongArrayValue(getValue()); + } + + long[] value = getValue(); + return new ClickHouseLongArrayValue(Arrays.copyOf(value, value.length)); + } + + @Override + public boolean isNullOrEmpty() { + return getValue().length == 0; + } + + @Override + + public ClickHouseLongArrayValue resetToNullOrEmpty() { + set(ClickHouseValues.EMPTY_LONG_ARRAY); + return this; + } + + @Override + public String toSqlExpression() { + long[] value = getValue(); + int len = value == null ? 0 : value.length; + if (len == 0) { + return ClickHouseValues.EMPTY_ARRAY_EXPR; + } + + StringBuilder builder = new StringBuilder().append('['); + for (int i = 0; i < len; i++) { + builder.append(value[i]).append(','); + } + builder.setLength(builder.length() - 1); + return builder.append(']').toString(); + } + + @Override + public ClickHouseLongArrayValue update(boolean[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + long[] v = new long[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i] ? 1L : 0L; + } + return set(v); + } + + @Override + public ClickHouseLongArrayValue update(char[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + long[] v = new long[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseLongArrayValue update(byte value) { + return set(new long[] { value }); + } + + @Override + public ClickHouseLongArrayValue update(byte[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + long[] v = new long[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseLongArrayValue update(short value) { + return set(new long[] { value }); + } + + @Override + public ClickHouseLongArrayValue update(short[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + long[] v = new long[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseLongArrayValue update(int value) { + return set(new long[] { value }); + } + + @Override + public ClickHouseLongArrayValue update(int[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + long[] v = new long[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseLongArrayValue update(long value) { + return set(new long[] { value }); + } + + @Override + public ClickHouseLongArrayValue update(long[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + return set(Arrays.copyOf(value, len)); + } + + @Override + public ClickHouseLongArrayValue update(float value) { + return set(new long[] { (long) value }); + } + + @Override + public ClickHouseLongArrayValue update(float[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + long[] v = new long[len]; + for (int i = 0; i < len; i++) { + v[i] = (long) value[i]; + } + return set(v); + } + + @Override + public ClickHouseLongArrayValue update(double value) { + return set(new long[] { (long) value }); + } + + @Override + public ClickHouseLongArrayValue update(double[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + long[] v = new long[len]; + for (int i = 0; i < len; i++) { + v[i] = (long) value[i]; + } + return set(v); + } + + @Override + public ClickHouseLongArrayValue update(BigInteger value) { + return set(value == null ? ClickHouseValues.EMPTY_LONG_ARRAY : new long[] { value.longValue() }); + } + + @Override + public ClickHouseLongArrayValue update(BigDecimal value) { + return set(value == null ? ClickHouseValues.EMPTY_LONG_ARRAY : new long[] { value.longValue() }); + } + + @Override + public ClickHouseLongArrayValue update(Enum value) { + return set(value == null ? ClickHouseValues.EMPTY_LONG_ARRAY : new long[] { value.ordinal() }); + } + + @Override + public ClickHouseLongArrayValue update(Inet4Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_IPV4, TYPE_NAME); + } + + @Override + public ClickHouseLongArrayValue update(Inet6Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_IPV6, TYPE_NAME); + } + + @Override + public ClickHouseLongArrayValue update(LocalDate value) { + return set(value == null ? ClickHouseValues.EMPTY_LONG_ARRAY : new long[] { value.toEpochDay() }); + } + + @Override + public ClickHouseLongArrayValue update(LocalTime value) { + return set(value == null ? ClickHouseValues.EMPTY_LONG_ARRAY : new long[] { value.toSecondOfDay() }); + } + + @Override + public ClickHouseLongArrayValue update(LocalDateTime value) { + return set( + value == null ? ClickHouseValues.EMPTY_LONG_ARRAY : new long[] { value.toEpochSecond(ZoneOffset.UTC) }); + } + + @Override + public ClickHouseLongArrayValue update(Collection value) { + int size = value == null ? 0 : value.size(); + if (size == 0) { + return resetToNullOrEmpty(); + } + + long[] v = new long[size]; + int index = 0; + for (Object o : value) { + v[index++] = o == null ? 0 : ((Number) o).longValue(); + } + return set(v); + } + + @Override + public ClickHouseLongArrayValue update(Enumeration value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + List v = new LinkedList<>(); + while (value.hasMoreElements()) { + v.add((Number) value.nextElement()); + } + + long[] values = new long[v.size()]; + int index = 0; + for (Number n : v) { + values[index++] = n == null ? 0 : n.longValue(); + } + return set(values); + } + + @Override + public ClickHouseLongArrayValue update(Map value) { + int size = value == null ? 0 : value.size(); + if (size == 0) { + return resetToNullOrEmpty(); + } + + long[] v = new long[size]; + int index = 0; + for (Entry e : value.entrySet()) { + Object o = e.getValue(); + v[index++] = o == null ? 0 : ((Number) e.getValue()).longValue(); + } + return set(v); + } + + @Override + public ClickHouseLongArrayValue update(String value) { + return set(new long[] { Long.parseLong(value) }); + } + + @Override + public ClickHouseLongArrayValue update(UUID value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_UUID, TYPE_NAME); + } + + @Override + public ClickHouseLongArrayValue update(ClickHouseValue value) { + if (value == null || value.isNullOrEmpty()) { + return resetToNullOrEmpty(); + } else if (value instanceof ClickHouseLongArrayValue) { + set(((ClickHouseLongArrayValue) value).getValue()); + } else { + update(value.asArray()); + } + return this; + } + + @Override + public ClickHouseLongArrayValue update(Object[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } else { + long[] values = new long[len]; + for (int i = 0; i < len; i++) { + Object o = value[i]; + values[i] = o == null ? 0 : ((Number) o).longValue(); + } + set(values); + } + + return this; + } + + @Override + public ClickHouseValue updateUnknown(Object value) { + if (value == null) { + return resetToNullOrEmpty(); + } else if (value instanceof Number) { + return set(new long[] { ((Number) value).longValue() }); + } else { + throw newUnsupportedException(value.getClass().getName(), TYPE_NAME); + } + } + + @Override + public ClickHouseLongArrayValue update(Object value) { + if (value instanceof long[]) { + set((long[]) value); + } else { + super.update(value); + } + return this; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { // too bad this is a mutable class :< + return true; + } else if (obj == null || getClass() != obj.getClass()) { + return false; + } + + return Arrays.equals(getValue(), ((ClickHouseLongArrayValue) obj).getValue()); + } + + @Override + public int hashCode() { + return Arrays.hashCode(getValue()); + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseShortArrayValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseShortArrayValue.java new file mode 100644 index 000000000..4daa591d4 --- /dev/null +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseShortArrayValue.java @@ -0,0 +1,479 @@ +package com.clickhouse.client.data.array; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.Collection; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.Map.Entry; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.client.data.ClickHouseObjectValue; + +/** + * Wrapper of {@code short[]}. + */ +public class ClickHouseShortArrayValue extends ClickHouseObjectValue { + private static final String TYPE_NAME = "short[]"; + + /** + * Creates an empty array. + * + * @return empty array + */ + + public static ClickHouseShortArrayValue ofEmpty() { + return of(ClickHouseValues.EMPTY_SHORT_ARRAY); + } + + /** + * Wrap the given value. + * + * @param value value + * @return object representing the value + */ + public static ClickHouseShortArrayValue of(short[] value) { + return of(null, value); + } + + /** + * Update value of the given object or create a new instance if {@code ref} is + * null. + * + * @param ref object to update, could be null + * @param value value + * @return same object as {@code ref} or a new instance if it's null + */ + + public static ClickHouseShortArrayValue of(ClickHouseValue ref, short[] value) { + return ref instanceof ClickHouseShortArrayValue ? ((ClickHouseShortArrayValue) ref).set(value) + : new ClickHouseShortArrayValue(value); + } + + protected ClickHouseShortArrayValue(short[] value) { + super(value); + } + + @Override + protected ClickHouseShortArrayValue set(short[] value) { + super.set(ClickHouseChecker.nonNull(value, ClickHouseValues.TYPE_ARRAY)); + return this; + } + + @Override + public Object[] asArray() { + short[] v = getValue(); + int len = v.length; + Short[] array = new Short[len]; + for (int i = 0; i < len; i++) { + array[i] = Short.valueOf(v[i]); + } + return array; + } + + @Override + public E[] asArray(Class clazz) { + short[] v = getValue(); + int len = v.length; + E[] array = ClickHouseValues.createObjectArray(clazz, len, 1); + for (int i = 0; i < len; i++) { + array[i] = clazz.cast(v[i]); + } + return array; + } + + @Override + public Map asMap(Class keyClass, Class valueClass) { + if (keyClass == null || valueClass == null) { + throw new IllegalArgumentException("Non-null key and value classes are required"); + } + short[] v = getValue(); + Map map = new LinkedHashMap<>(); + for (int i = 0; i < v.length; i++) { + map.put(keyClass.cast(i + 1), valueClass.cast(v[i])); + } + // why not use Collections.unmodifiableMap(map) here? + return map; + } + + @Override + public String asString(int length, Charset charset) { + String str = Arrays.toString(getValue()); + if (length > 0) { + ClickHouseChecker.notWithDifferentLength(str.getBytes(charset == null ? StandardCharsets.UTF_8 : charset), + length); + } + + return str; + } + + @Override + public ClickHouseShortArrayValue copy(boolean deep) { + if (!deep) { + return new ClickHouseShortArrayValue(getValue()); + } + + short[] value = getValue(); + return new ClickHouseShortArrayValue(Arrays.copyOf(value, value.length)); + } + + @Override + public boolean isNullOrEmpty() { + return getValue().length == 0; + } + + @Override + + public ClickHouseShortArrayValue resetToNullOrEmpty() { + set(ClickHouseValues.EMPTY_SHORT_ARRAY); + return this; + } + + @Override + public String toSqlExpression() { + short[] value = getValue(); + int len = value == null ? 0 : value.length; + if (len == 0) { + return ClickHouseValues.EMPTY_ARRAY_EXPR; + } + + StringBuilder builder = new StringBuilder().append('['); + for (int i = 0; i < len; i++) { + builder.append(value[i]).append(','); + } + builder.setLength(builder.length() - 1); + return builder.append(']').toString(); + } + + @Override + public ClickHouseShortArrayValue update(boolean[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + short[] v = new short[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i] ? (short) 1 : (short) 0; + } + return set(v); + } + + @Override + public ClickHouseShortArrayValue update(char[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + short[] v = new short[len]; + for (int i = 0; i < len; i++) { + v[i] = (short) value[i]; + } + return set(v); + } + + @Override + public ClickHouseShortArrayValue update(byte value) { + return set(new short[] { value }); + } + + @Override + public ClickHouseShortArrayValue update(byte[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + short[] v = new short[len]; + for (int i = 0; i < len; i++) { + v[i] = value[i]; + } + return set(v); + } + + @Override + public ClickHouseShortArrayValue update(short value) { + return set(new short[] { value }); + } + + @Override + public ClickHouseShortArrayValue update(short[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + return set(Arrays.copyOf(value, len)); + } + + @Override + public ClickHouseShortArrayValue update(int value) { + return set(new short[] { (short) value }); + } + + @Override + public ClickHouseShortArrayValue update(int[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + short[] v = new short[len]; + for (int i = 0; i < len; i++) { + v[i] = (short) value[i]; + } + return set(v); + } + + @Override + public ClickHouseShortArrayValue update(long value) { + return set(new short[] { (short) value }); + } + + @Override + public ClickHouseShortArrayValue update(long[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + short[] v = new short[len]; + for (int i = 0; i < len; i++) { + v[i] = (short) value[i]; + } + return set(v); + } + + @Override + public ClickHouseShortArrayValue update(float value) { + return set(new short[] { (short) value }); + } + + @Override + public ClickHouseShortArrayValue update(float[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + short[] v = new short[len]; + for (int i = 0; i < len; i++) { + v[i] = (short) value[i]; + } + return set(v); + } + + @Override + public ClickHouseShortArrayValue update(double value) { + return set(new short[] { (short) value }); + } + + @Override + public ClickHouseShortArrayValue update(double[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } + + short[] v = new short[len]; + for (int i = 0; i < len; i++) { + v[i] = (short) value[i]; + } + return set(v); + } + + @Override + public ClickHouseShortArrayValue update(BigInteger value) { + return set(value == null ? ClickHouseValues.EMPTY_SHORT_ARRAY : new short[] { value.shortValue() }); + } + + @Override + public ClickHouseShortArrayValue update(BigDecimal value) { + return set(value == null ? ClickHouseValues.EMPTY_SHORT_ARRAY : new short[] { value.shortValue() }); + } + + @Override + public ClickHouseShortArrayValue update(Enum value) { + return set(value == null ? ClickHouseValues.EMPTY_SHORT_ARRAY : new short[] { (short) value.ordinal() }); + } + + @Override + public ClickHouseShortArrayValue update(Inet4Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_IPV4, TYPE_NAME); + } + + @Override + public ClickHouseShortArrayValue update(Inet6Address value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_IPV6, TYPE_NAME); + } + + @Override + public ClickHouseShortArrayValue update(LocalDate value) { + return set(value == null ? ClickHouseValues.EMPTY_SHORT_ARRAY : new short[] { (short) value.toEpochDay() }); + } + + @Override + public ClickHouseShortArrayValue update(LocalTime value) { + return set(value == null ? ClickHouseValues.EMPTY_SHORT_ARRAY : new short[] { (short) value.toSecondOfDay() }); + } + + @Override + public ClickHouseShortArrayValue update(LocalDateTime value) { + return set(value == null ? ClickHouseValues.EMPTY_SHORT_ARRAY + : new short[] { (short) value.toEpochSecond(ZoneOffset.UTC) }); + } + + @Override + public ClickHouseShortArrayValue update(Collection value) { + int size = value == null ? 0 : value.size(); + if (size == 0) { + return resetToNullOrEmpty(); + } + + short[] v = new short[size]; + int index = 0; + for (Object o : value) { + v[index++] = o == null ? 0 : ((Number) o).shortValue(); + } + return set(v); + } + + @Override + public ClickHouseShortArrayValue update(Enumeration value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + List v = new LinkedList<>(); + while (value.hasMoreElements()) { + v.add((Number) value.nextElement()); + } + + short[] values = new short[v.size()]; + int index = 0; + for (Number n : v) { + values[index++] = n == null ? 0 : n.shortValue(); + } + return set(values); + } + + @Override + public ClickHouseShortArrayValue update(Map value) { + int size = value == null ? 0 : value.size(); + if (size == 0) { + return resetToNullOrEmpty(); + } + + short[] v = new short[size]; + int index = 0; + for (Entry e : value.entrySet()) { + Object o = e.getValue(); + v[index++] = o == null ? 0 : ((Number) e.getValue()).shortValue(); + } + return set(v); + } + + @Override + public ClickHouseShortArrayValue update(String value) { + return set(new short[] { Short.parseShort(value) }); + } + + @Override + public ClickHouseShortArrayValue update(UUID value) { + if (value == null) { + return resetToNullOrEmpty(); + } + + throw newUnsupportedException(ClickHouseValues.TYPE_UUID, TYPE_NAME); + } + + @Override + public ClickHouseShortArrayValue update(ClickHouseValue value) { + if (value == null || value.isNullOrEmpty()) { + return resetToNullOrEmpty(); + } else if (value instanceof ClickHouseShortArrayValue) { + set(((ClickHouseShortArrayValue) value).getValue()); + } else { + update(value.asArray()); + } + return this; + } + + @Override + public ClickHouseShortArrayValue update(Object[] value) { + int len = value == null ? 0 : value.length; + if (len == 0) { + return resetToNullOrEmpty(); + } else { + short[] values = new short[len]; + for (int i = 0; i < len; i++) { + Object o = value[i]; + values[i] = o == null ? 0 : ((Number) o).shortValue(); + } + set(values); + } + + return this; + } + + @Override + public ClickHouseValue updateUnknown(Object value) { + if (value == null) { + return resetToNullOrEmpty(); + } else if (value instanceof Number) { + return set(new short[] { ((Number) value).shortValue() }); + } else { + throw newUnsupportedException(value.getClass().getName(), TYPE_NAME); + } + } + + @Override + public ClickHouseShortArrayValue update(Object value) { + if (value instanceof short[]) { + set((short[]) value); + } else { + super.update(value); + } + return this; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { // too bad this is a mutable class :< + return true; + } else if (obj == null || getClass() != obj.getClass()) { + return false; + } + + return Arrays.equals(getValue(), ((ClickHouseShortArrayValue) obj).getValue()); + } + + @Override + public int hashCode() { + return Arrays.hashCode(getValue()); + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/exception/ClickHouseErrorCode.java b/clickhouse-client/src/main/java/com/clickhouse/client/exception/ClickHouseErrorCode.java deleted file mode 100644 index 34acdf0ff..000000000 --- a/clickhouse-client/src/main/java/com/clickhouse/client/exception/ClickHouseErrorCode.java +++ /dev/null @@ -1,622 +0,0 @@ -package com.clickhouse.client.exception; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** - * All supported error codes defined in ClickHouse. - */ -public enum ClickHouseErrorCode { - // grep -n '^[[:space:]]*M(' | sed -e - // 's|^\([0-9]*\):[[:space:]]*M(\([0-9]*\),[[:space:]]*\([0-9A-Z_]*\)).*|\3(\2), - // // - // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L\1|g' - OK(0), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L17 - UNSUPPORTED_METHOD(1), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L18 - UNSUPPORTED_PARAMETER(2), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L19 - UNEXPECTED_END_OF_FILE(3), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L20 - EXPECTED_END_OF_FILE(4), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L21 - CANNOT_PARSE_TEXT(6), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L22 - INCORRECT_NUMBER_OF_COLUMNS(7), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L23 - THERE_IS_NO_COLUMN(8), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L24 - SIZES_OF_COLUMNS_DOESNT_MATCH(9), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L25 - NOT_FOUND_COLUMN_IN_BLOCK(10), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L26 - POSITION_OUT_OF_BOUND(11), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L27 - PARAMETER_OUT_OF_BOUND(12), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L28 - SIZES_OF_COLUMNS_IN_TUPLE_DOESNT_MATCH(13), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L29 - DUPLICATE_COLUMN(15), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L30 - NO_SUCH_COLUMN_IN_TABLE(16), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L31 - DELIMITER_IN_STRING_LITERAL_DOESNT_MATCH(17), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L32 - CANNOT_INSERT_ELEMENT_INTO_CONSTANT_COLUMN(18), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L33 - SIZE_OF_FIXED_STRING_DOESNT_MATCH(19), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L34 - NUMBER_OF_COLUMNS_DOESNT_MATCH(20), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L35 - CANNOT_READ_ALL_DATA_FROM_TAB_SEPARATED_INPUT(21), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L36 - CANNOT_PARSE_ALL_VALUE_FROM_TAB_SEPARATED_INPUT(22), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L37 - CANNOT_READ_FROM_ISTREAM(23), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L38 - CANNOT_WRITE_TO_OSTREAM(24), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L39 - CANNOT_PARSE_ESCAPE_SEQUENCE(25), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L40 - CANNOT_PARSE_QUOTED_STRING(26), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L41 - CANNOT_PARSE_INPUT_ASSERTION_FAILED(27), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L42 - CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER(28), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L43 - CANNOT_PRINT_INTEGER(29), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L44 - CANNOT_READ_SIZE_OF_COMPRESSED_CHUNK(30), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L45 - CANNOT_READ_COMPRESSED_CHUNK(31), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L46 - ATTEMPT_TO_READ_AFTER_EOF(32), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L47 - CANNOT_READ_ALL_DATA(33), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L48 - TOO_MANY_ARGUMENTS_FOR_FUNCTION(34), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L49 - TOO_FEW_ARGUMENTS_FOR_FUNCTION(35), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L50 - BAD_ARGUMENTS(36), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L51 - UNKNOWN_ELEMENT_IN_AST(37), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L52 - CANNOT_PARSE_DATE(38), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L53 - TOO_LARGE_SIZE_COMPRESSED(39), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L54 - CHECKSUM_DOESNT_MATCH(40), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L55 - CANNOT_PARSE_DATETIME(41), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L56 - NUMBER_OF_ARGUMENTS_DOESNT_MATCH(42), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L57 - ILLEGAL_TYPE_OF_ARGUMENT(43), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L58 - ILLEGAL_COLUMN(44), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L59 - ILLEGAL_NUMBER_OF_RESULT_COLUMNS(45), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L60 - UNKNOWN_FUNCTION(46), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L61 - UNKNOWN_IDENTIFIER(47), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L62 - NOT_IMPLEMENTED(48), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L63 - LOGICAL_ERROR(49), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L64 - UNKNOWN_TYPE(50), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L65 - EMPTY_LIST_OF_COLUMNS_QUERIED(51), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L66 - COLUMN_QUERIED_MORE_THAN_ONCE(52), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L67 - TYPE_MISMATCH(53), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L68 - STORAGE_DOESNT_ALLOW_PARAMETERS(54), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L69 - STORAGE_REQUIRES_PARAMETER(55), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L70 - UNKNOWN_STORAGE(56), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L71 - TABLE_ALREADY_EXISTS(57), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L72 - TABLE_METADATA_ALREADY_EXISTS(58), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L73 - ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER(59), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L74 - UNKNOWN_TABLE(60), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L75 - ONLY_FILTER_COLUMN_IN_BLOCK(61), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L76 - SYNTAX_ERROR(62), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L77 - UNKNOWN_AGGREGATE_FUNCTION(63), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L78 - CANNOT_READ_AGGREGATE_FUNCTION_FROM_TEXT(64), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L79 - CANNOT_WRITE_AGGREGATE_FUNCTION_AS_TEXT(65), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L80 - NOT_A_COLUMN(66), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L81 - ILLEGAL_KEY_OF_AGGREGATION(67), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L82 - CANNOT_GET_SIZE_OF_FIELD(68), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L83 - ARGUMENT_OUT_OF_BOUND(69), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L84 - CANNOT_CONVERT_TYPE(70), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L85 - CANNOT_WRITE_AFTER_END_OF_BUFFER(71), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L86 - CANNOT_PARSE_NUMBER(72), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L87 - UNKNOWN_FORMAT(73), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L88 - CANNOT_READ_FROM_FILE_DESCRIPTOR(74), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L89 - CANNOT_WRITE_TO_FILE_DESCRIPTOR(75), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L90 - CANNOT_OPEN_FILE(76), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L91 - CANNOT_CLOSE_FILE(77), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L92 - UNKNOWN_TYPE_OF_QUERY(78), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L93 - INCORRECT_FILE_NAME(79), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L94 - INCORRECT_QUERY(80), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L95 - UNKNOWN_DATABASE(81), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L96 - DATABASE_ALREADY_EXISTS(82), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L97 - DIRECTORY_DOESNT_EXIST(83), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L98 - DIRECTORY_ALREADY_EXISTS(84), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L99 - FORMAT_IS_NOT_SUITABLE_FOR_INPUT(85), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L100 - RECEIVED_ERROR_FROM_REMOTE_IO_SERVER(86), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L101 - CANNOT_SEEK_THROUGH_FILE(87), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L102 - CANNOT_TRUNCATE_FILE(88), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L103 - UNKNOWN_COMPRESSION_METHOD(89), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L104 - EMPTY_LIST_OF_COLUMNS_PASSED(90), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L105 - SIZES_OF_MARKS_FILES_ARE_INCONSISTENT(91), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L106 - EMPTY_DATA_PASSED(92), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L107 - UNKNOWN_AGGREGATED_DATA_VARIANT(93), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L108 - CANNOT_MERGE_DIFFERENT_AGGREGATED_DATA_VARIANTS(94), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L109 - CANNOT_READ_FROM_SOCKET(95), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L110 - CANNOT_WRITE_TO_SOCKET(96), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L111 - CANNOT_READ_ALL_DATA_FROM_CHUNKED_INPUT(97), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L112 - CANNOT_WRITE_TO_EMPTY_BLOCK_OUTPUT_STREAM(98), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L113 - UNKNOWN_PACKET_FROM_CLIENT(99), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L114 - UNKNOWN_PACKET_FROM_SERVER(100), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L115 - UNEXPECTED_PACKET_FROM_CLIENT(101), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L116 - UNEXPECTED_PACKET_FROM_SERVER(102), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L117 - RECEIVED_DATA_FOR_WRONG_QUERY_ID(103), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L118 - TOO_SMALL_BUFFER_SIZE(104), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L119 - CANNOT_READ_HISTORY(105), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L120 - CANNOT_APPEND_HISTORY(106), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L121 - FILE_DOESNT_EXIST(107), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L122 - NO_DATA_TO_INSERT(108), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L123 - CANNOT_BLOCK_SIGNAL(109), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L124 - CANNOT_UNBLOCK_SIGNAL(110), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L125 - CANNOT_MANIPULATE_SIGSET(111), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L126 - CANNOT_WAIT_FOR_SIGNAL(112), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L127 - THERE_IS_NO_SESSION(113), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L128 - CANNOT_CLOCK_GETTIME(114), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L129 - UNKNOWN_SETTING(115), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L130 - THERE_IS_NO_DEFAULT_VALUE(116), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L131 - INCORRECT_DATA(117), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L132 - ENGINE_REQUIRED(119), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L133 - CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE(120), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L134 - UNSUPPORTED_JOIN_KEYS(121), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L135 - INCOMPATIBLE_COLUMNS(122), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L136 - UNKNOWN_TYPE_OF_AST_NODE(123), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L137 - INCORRECT_ELEMENT_OF_SET(124), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L138 - INCORRECT_RESULT_OF_SCALAR_SUBQUERY(125), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L139 - CANNOT_GET_RETURN_TYPE(126), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L140 - ILLEGAL_INDEX(127), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L141 - TOO_LARGE_ARRAY_SIZE(128), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L142 - FUNCTION_IS_SPECIAL(129), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L143 - CANNOT_READ_ARRAY_FROM_TEXT(130), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L144 - TOO_LARGE_STRING_SIZE(131), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L145 - AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS(133), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L146 - PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS(134), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L147 - ZERO_ARRAY_OR_TUPLE_INDEX(135), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L148 - UNKNOWN_ELEMENT_IN_CONFIG(137), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L149 - EXCESSIVE_ELEMENT_IN_CONFIG(138), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L150 - NO_ELEMENTS_IN_CONFIG(139), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L151 - ALL_REQUESTED_COLUMNS_ARE_MISSING(140), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L152 - SAMPLING_NOT_SUPPORTED(141), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L153 - NOT_FOUND_NODE(142), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L154 - FOUND_MORE_THAN_ONE_NODE(143), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L155 - FIRST_DATE_IS_BIGGER_THAN_LAST_DATE(144), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L156 - UNKNOWN_OVERFLOW_MODE(145), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L157 - QUERY_SECTION_DOESNT_MAKE_SENSE(146), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L158 - NOT_FOUND_FUNCTION_ELEMENT_FOR_AGGREGATE(147), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L159 - NOT_FOUND_RELATION_ELEMENT_FOR_CONDITION(148), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L160 - NOT_FOUND_RHS_ELEMENT_FOR_CONDITION(149), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L161 - EMPTY_LIST_OF_ATTRIBUTES_PASSED(150), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L162 - INDEX_OF_COLUMN_IN_SORT_CLAUSE_IS_OUT_OF_RANGE(151), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L163 - UNKNOWN_DIRECTION_OF_SORTING(152), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L164 - ILLEGAL_DIVISION(153), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L165 - AGGREGATE_FUNCTION_NOT_APPLICABLE(154), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L166 - UNKNOWN_RELATION(155), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L167 - DICTIONARIES_WAS_NOT_LOADED(156), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L168 - ILLEGAL_OVERFLOW_MODE(157), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L169 - TOO_MANY_ROWS(158), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L170 - TIMEOUT_EXCEEDED(159), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L171 - TOO_SLOW(160), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L172 - TOO_MANY_COLUMNS(161), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L173 - TOO_DEEP_SUBQUERIES(162), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L174 - TOO_DEEP_PIPELINE(163), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L175 - READONLY(164), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L176 - TOO_MANY_TEMPORARY_COLUMNS(165), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L177 - TOO_MANY_TEMPORARY_NON_CONST_COLUMNS(166), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L178 - TOO_DEEP_AST(167), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L179 - TOO_BIG_AST(168), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L180 - BAD_TYPE_OF_FIELD(169), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L181 - BAD_GET(170), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L182 - CANNOT_CREATE_DIRECTORY(172), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L183 - CANNOT_ALLOCATE_MEMORY(173), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L184 - CYCLIC_ALIASES(174), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L185 - CHUNK_NOT_FOUND(176), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L186 - DUPLICATE_CHUNK_NAME(177), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L187 - MULTIPLE_ALIASES_FOR_EXPRESSION(178), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L188 - MULTIPLE_EXPRESSIONS_FOR_ALIAS(179), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L189 - THERE_IS_NO_PROFILE(180), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L190 - ILLEGAL_FINAL(181), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L191 - ILLEGAL_PREWHERE(182), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L192 - UNEXPECTED_EXPRESSION(183), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L193 - ILLEGAL_AGGREGATION(184), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L194 - UNSUPPORTED_MYISAM_BLOCK_TYPE(185), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L195 - UNSUPPORTED_COLLATION_LOCALE(186), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L196 - COLLATION_COMPARISON_FAILED(187), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L197 - UNKNOWN_ACTION(188), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L198 - TABLE_MUST_NOT_BE_CREATED_MANUALLY(189), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L199 - SIZES_OF_ARRAYS_DOESNT_MATCH(190), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L200 - SET_SIZE_LIMIT_EXCEEDED(191), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L201 - UNKNOWN_USER(192), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L202 - WRONG_PASSWORD(193), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L203 - REQUIRED_PASSWORD(194), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L204 - IP_ADDRESS_NOT_ALLOWED(195), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L205 - UNKNOWN_ADDRESS_PATTERN_TYPE(196), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L206 - SERVER_REVISION_IS_TOO_OLD(197), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L207 - DNS_ERROR(198), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L208 - UNKNOWN_QUOTA(199), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L209 - QUOTA_DOESNT_ALLOW_KEYS(200), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L210 - QUOTA_EXPIRED(201), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L211 - TOO_MANY_SIMULTANEOUS_QUERIES(202), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L212 - NO_FREE_CONNECTION(203), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L213 - CANNOT_FSYNC(204), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L214 - NESTED_TYPE_TOO_DEEP(205), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L215 - ALIAS_REQUIRED(206), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L216 - AMBIGUOUS_IDENTIFIER(207), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L217 - EMPTY_NESTED_TABLE(208), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L218 - SOCKET_TIMEOUT(209), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L219 - NETWORK_ERROR(210), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L220 - EMPTY_QUERY(211), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L221 - UNKNOWN_LOAD_BALANCING(212), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L222 - UNKNOWN_TOTALS_MODE(213), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L223 - CANNOT_STATVFS(214), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L224 - NOT_AN_AGGREGATE(215), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L225 - QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING(216), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L226 - CLIENT_HAS_CONNECTED_TO_WRONG_PORT(217), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L227 - TABLE_IS_DROPPED(218), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L228 - DATABASE_NOT_EMPTY(219), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L229 - DUPLICATE_INTERSERVER_IO_ENDPOINT(220), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L230 - NO_SUCH_INTERSERVER_IO_ENDPOINT(221), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L231 - ADDING_REPLICA_TO_NON_EMPTY_TABLE(222), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L232 - UNEXPECTED_AST_STRUCTURE(223), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L233 - REPLICA_IS_ALREADY_ACTIVE(224), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L234 - NO_ZOOKEEPER(225), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L235 - NO_FILE_IN_DATA_PART(226), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L236 - UNEXPECTED_FILE_IN_DATA_PART(227), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L237 - BAD_SIZE_OF_FILE_IN_DATA_PART(228), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L238 - QUERY_IS_TOO_LARGE(229), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L239 - NOT_FOUND_EXPECTED_DATA_PART(230), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L240 - TOO_MANY_UNEXPECTED_DATA_PARTS(231), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L241 - NO_SUCH_DATA_PART(232), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L242 - BAD_DATA_PART_NAME(233), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L243 - NO_REPLICA_HAS_PART(234), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L244 - DUPLICATE_DATA_PART(235), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L245 - ABORTED(236), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L246 - NO_REPLICA_NAME_GIVEN(237), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L247 - FORMAT_VERSION_TOO_OLD(238), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L248 - CANNOT_MUNMAP(239), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L249 - CANNOT_MREMAP(240), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L250 - MEMORY_LIMIT_EXCEEDED(241), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L251 - TABLE_IS_READ_ONLY(242), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L252 - NOT_ENOUGH_SPACE(243), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L253 - UNEXPECTED_ZOOKEEPER_ERROR(244), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L254 - CORRUPTED_DATA(246), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L255 - INCORRECT_MARK(247), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L256 - INVALID_PARTITION_VALUE(248), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L257 - NOT_ENOUGH_BLOCK_NUMBERS(250), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L258 - NO_SUCH_REPLICA(251), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L259 - TOO_MANY_PARTS(252), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L260 - REPLICA_IS_ALREADY_EXIST(253), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L261 - NO_ACTIVE_REPLICAS(254), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L262 - TOO_MANY_RETRIES_TO_FETCH_PARTS(255), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L263 - PARTITION_ALREADY_EXISTS(256), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L264 - PARTITION_DOESNT_EXIST(257), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L265 - UNION_ALL_RESULT_STRUCTURES_MISMATCH(258), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L266 - CLIENT_OUTPUT_FORMAT_SPECIFIED(260), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L267 - UNKNOWN_BLOCK_INFO_FIELD(261), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L268 - BAD_COLLATION(262), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L269 - CANNOT_COMPILE_CODE(263), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L270 - INCOMPATIBLE_TYPE_OF_JOIN(264), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L271 - NO_AVAILABLE_REPLICA(265), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L272 - MISMATCH_REPLICAS_DATA_SOURCES(266), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L273 - STORAGE_DOESNT_SUPPORT_PARALLEL_REPLICAS(267), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L274 - CPUID_ERROR(268), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L275 - INFINITE_LOOP(269), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L276 - CANNOT_COMPRESS(270), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L277 - CANNOT_DECOMPRESS(271), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L278 - CANNOT_IO_SUBMIT(272), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L279 - CANNOT_IO_GETEVENTS(273), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L280 - AIO_READ_ERROR(274), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L281 - AIO_WRITE_ERROR(275), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L282 - INDEX_NOT_USED(277), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L283 - ALL_CONNECTION_TRIES_FAILED(279), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L284 - NO_AVAILABLE_DATA(280), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L285 - DICTIONARY_IS_EMPTY(281), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L286 - INCORRECT_INDEX(282), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L287 - UNKNOWN_DISTRIBUTED_PRODUCT_MODE(283), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L288 - WRONG_GLOBAL_SUBQUERY(284), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L289 - TOO_FEW_LIVE_REPLICAS(285), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L290 - UNSATISFIED_QUORUM_FOR_PREVIOUS_WRITE(286), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L291 - UNKNOWN_FORMAT_VERSION(287), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L292 - DISTRIBUTED_IN_JOIN_SUBQUERY_DENIED(288), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L293 - REPLICA_IS_NOT_IN_QUORUM(289), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L294 - LIMIT_EXCEEDED(290), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L295 - DATABASE_ACCESS_DENIED(291), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L296 - MONGODB_CANNOT_AUTHENTICATE(293), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L297 - INVALID_BLOCK_EXTRA_INFO(294), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L298 - RECEIVED_EMPTY_DATA(295), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L299 - NO_REMOTE_SHARD_FOUND(296), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L300 - SHARD_HAS_NO_CONNECTIONS(297), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L301 - CANNOT_PIPE(298), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L302 - CANNOT_FORK(299), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L303 - CANNOT_DLSYM(300), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L304 - CANNOT_CREATE_CHILD_PROCESS(301), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L305 - CHILD_WAS_NOT_EXITED_NORMALLY(302), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L306 - CANNOT_SELECT(303), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L307 - CANNOT_WAITPID(304), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L308 - TABLE_WAS_NOT_DROPPED(305), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L309 - TOO_DEEP_RECURSION(306), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L310 - TOO_MANY_BYTES(307), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L311 - UNEXPECTED_NODE_IN_ZOOKEEPER(308), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L312 - FUNCTION_CANNOT_HAVE_PARAMETERS(309), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L313 - INVALID_SHARD_WEIGHT(317), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L314 - INVALID_CONFIG_PARAMETER(318), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L315 - UNKNOWN_STATUS_OF_INSERT(319), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L316 - VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE(321), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L317 - BARRIER_TIMEOUT(335), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L318 - UNKNOWN_DATABASE_ENGINE(336), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L319 - DDL_GUARD_IS_ACTIVE(337), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L320 - UNFINISHED(341), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L321 - METADATA_MISMATCH(342), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L322 - SUPPORT_IS_DISABLED(344), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L323 - TABLE_DIFFERS_TOO_MUCH(345), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L324 - CANNOT_CONVERT_CHARSET(346), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L325 - CANNOT_LOAD_CONFIG(347), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L326 - CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN(349), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L327 - INCOMPATIBLE_SOURCE_TABLES(350), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L328 - AMBIGUOUS_TABLE_NAME(351), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L329 - AMBIGUOUS_COLUMN_NAME(352), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L330 - INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE(353), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L331 - ZLIB_INFLATE_FAILED(354), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L332 - ZLIB_DEFLATE_FAILED(355), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L333 - BAD_LAMBDA(356), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L334 - RESERVED_IDENTIFIER_NAME(357), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L335 - INTO_OUTFILE_NOT_ALLOWED(358), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L336 - TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT(359), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L337 - CANNOT_CREATE_CHARSET_CONVERTER(360), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L338 - SEEK_POSITION_OUT_OF_BOUND(361), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L339 - CURRENT_WRITE_BUFFER_IS_EXHAUSTED(362), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L340 - CANNOT_CREATE_IO_BUFFER(363), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L341 - RECEIVED_ERROR_TOO_MANY_REQUESTS(364), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L342 - SIZES_OF_NESTED_COLUMNS_ARE_INCONSISTENT(366), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L343 - TOO_MANY_FETCHES(367), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L344 - ALL_REPLICAS_ARE_STALE(369), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L345 - DATA_TYPE_CANNOT_BE_USED_IN_TABLES(370), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L346 - INCONSISTENT_CLUSTER_DEFINITION(371), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L347 - SESSION_NOT_FOUND(372), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L348 - SESSION_IS_LOCKED(373), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L349 - INVALID_SESSION_TIMEOUT(374), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L350 - CANNOT_DLOPEN(375), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L351 - CANNOT_PARSE_UUID(376), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L352 - ILLEGAL_SYNTAX_FOR_DATA_TYPE(377), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L353 - DATA_TYPE_CANNOT_HAVE_ARGUMENTS(378), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L354 - UNKNOWN_STATUS_OF_DISTRIBUTED_DDL_TASK(379), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L355 - CANNOT_KILL(380), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L356 - HTTP_LENGTH_REQUIRED(381), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L357 - CANNOT_LOAD_CATBOOST_MODEL(382), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L358 - CANNOT_APPLY_CATBOOST_MODEL(383), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L359 - PART_IS_TEMPORARILY_LOCKED(384), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L360 - MULTIPLE_STREAMS_REQUIRED(385), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L361 - NO_COMMON_TYPE(386), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L362 - DICTIONARY_ALREADY_EXISTS(387), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L363 - CANNOT_ASSIGN_OPTIMIZE(388), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L364 - INSERT_WAS_DEDUPLICATED(389), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L365 - CANNOT_GET_CREATE_TABLE_QUERY(390), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L366 - EXTERNAL_LIBRARY_ERROR(391), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L367 - QUERY_IS_PROHIBITED(392), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L368 - THERE_IS_NO_QUERY(393), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L369 - QUERY_WAS_CANCELLED(394), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L370 - FUNCTION_THROW_IF_VALUE_IS_NON_ZERO(395), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L371 - TOO_MANY_ROWS_OR_BYTES(396), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L372 - QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW(397), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L373 - UNKNOWN_MUTATION_COMMAND(398), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L374 - FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT(399), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L375 - CANNOT_STAT(400), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L376 - FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME(401), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L377 - CANNOT_IOSETUP(402), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L378 - INVALID_JOIN_ON_EXPRESSION(403), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L379 - BAD_ODBC_CONNECTION_STRING(404), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L380 - PARTITION_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT(405), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L381 - TOP_AND_LIMIT_TOGETHER(406), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L382 - DECIMAL_OVERFLOW(407), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L383 - BAD_REQUEST_PARAMETER(408), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L384 - EXTERNAL_EXECUTABLE_NOT_FOUND(409), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L385 - EXTERNAL_SERVER_IS_NOT_RESPONDING(410), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L386 - PTHREAD_ERROR(411), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L387 - NETLINK_ERROR(412), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L388 - CANNOT_SET_SIGNAL_HANDLER(413), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L389 - ALL_REPLICAS_LOST(415), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L390 - REPLICA_STATUS_CHANGED(416), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L391 - EXPECTED_ALL_OR_ANY(417), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L392 - UNKNOWN_JOIN(418), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L393 - MULTIPLE_ASSIGNMENTS_TO_COLUMN(419), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L394 - CANNOT_UPDATE_COLUMN(420), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L395 - CANNOT_ADD_DIFFERENT_AGGREGATE_STATES(421), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L396 - UNSUPPORTED_URI_SCHEME(422), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L397 - CANNOT_GETTIMEOFDAY(423), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L398 - CANNOT_LINK(424), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L399 - SYSTEM_ERROR(425), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L400 - CANNOT_COMPILE_REGEXP(427), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L401 - UNKNOWN_LOG_LEVEL(428), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L402 - FAILED_TO_GETPWUID(429), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L403 - MISMATCHING_USERS_FOR_PROCESS_AND_DATA(430), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L404 - ILLEGAL_SYNTAX_FOR_CODEC_TYPE(431), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L405 - UNKNOWN_CODEC(432), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L406 - ILLEGAL_CODEC_PARAMETER(433), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L407 - CANNOT_PARSE_PROTOBUF_SCHEMA(434), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L408 - NO_COLUMN_SERIALIZED_TO_REQUIRED_PROTOBUF_FIELD(435), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L409 - PROTOBUF_BAD_CAST(436), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L410 - PROTOBUF_FIELD_NOT_REPEATED(437), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L411 - DATA_TYPE_CANNOT_BE_PROMOTED(438), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L412 - CANNOT_SCHEDULE_TASK(439), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L413 - INVALID_LIMIT_EXPRESSION(440), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L414 - CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING(441), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L415 - BAD_DATABASE_FOR_TEMPORARY_TABLE(442), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L416 - NO_COLUMNS_SERIALIZED_TO_PROTOBUF_FIELDS(443), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L417 - UNKNOWN_PROTOBUF_FORMAT(444), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L418 - CANNOT_MPROTECT(445), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L419 - FUNCTION_NOT_ALLOWED(446), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L420 - HYPERSCAN_CANNOT_SCAN_TEXT(447), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L421 - BROTLI_READ_FAILED(448), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L422 - BROTLI_WRITE_FAILED(449), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L423 - BAD_TTL_EXPRESSION(450), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L424 - BAD_TTL_FILE(451), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L425 - SETTING_CONSTRAINT_VIOLATION(452), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L426 - MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES(453), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L427 - OPENSSL_ERROR(454), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L428 - SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY(455), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L429 - UNKNOWN_QUERY_PARAMETER(456), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L430 - BAD_QUERY_PARAMETER(457), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L431 - CANNOT_UNLINK(458), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L432 - CANNOT_SET_THREAD_PRIORITY(459), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L433 - CANNOT_CREATE_TIMER(460), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L434 - CANNOT_SET_TIMER_PERIOD(461), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L435 - CANNOT_DELETE_TIMER(462), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L436 - CANNOT_FCNTL(463), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L437 - CANNOT_PARSE_ELF(464), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L438 - CANNOT_PARSE_DWARF(465), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L439 - INSECURE_PATH(466), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L440 - CANNOT_PARSE_BOOL(467), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L441 - CANNOT_PTHREAD_ATTR(468), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L442 - VIOLATED_CONSTRAINT(469), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L443 - QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW(470), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L444 - INVALID_SETTING_VALUE(471), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L445 - READONLY_SETTING(472), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L446 - DEADLOCK_AVOIDED(473), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L447 - INVALID_TEMPLATE_FORMAT(474), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L448 - INVALID_WITH_FILL_EXPRESSION(475), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L449 - WITH_TIES_WITHOUT_ORDER_BY(476), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L450 - INVALID_USAGE_OF_INPUT(477), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L451 - UNKNOWN_POLICY(478), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L452 - UNKNOWN_DISK(479), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L453 - UNKNOWN_PROTOCOL(480), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L454 - PATH_ACCESS_DENIED(481), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L455 - DICTIONARY_ACCESS_DENIED(482), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L456 - TOO_MANY_REDIRECTS(483), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L457 - INTERNAL_REDIS_ERROR(484), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L458 - SCALAR_ALREADY_EXISTS(485), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L459 - CANNOT_GET_CREATE_DICTIONARY_QUERY(487), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L460 - UNKNOWN_DICTIONARY(488), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L461 - INCORRECT_DICTIONARY_DEFINITION(489), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L462 - CANNOT_FORMAT_DATETIME(490), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L463 - UNACCEPTABLE_URL(491), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L464 - ACCESS_ENTITY_NOT_FOUND(492), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L465 - ACCESS_ENTITY_ALREADY_EXISTS(493), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L466 - ACCESS_ENTITY_FOUND_DUPLICATES(494), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L467 - ACCESS_STORAGE_READONLY(495), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L468 - QUOTA_REQUIRES_CLIENT_KEY(496), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L469 - ACCESS_DENIED(497), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L470 - LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED(498), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L471 - S3_ERROR(499), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L472 - CANNOT_CREATE_DATABASE(501), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L473 - CANNOT_SIGQUEUE(502), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L474 - AGGREGATE_FUNCTION_THROW(503), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L475 - FILE_ALREADY_EXISTS(504), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L476 - CANNOT_DELETE_DIRECTORY(505), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L477 - UNEXPECTED_ERROR_CODE(506), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L478 - UNABLE_TO_SKIP_UNUSED_SHARDS(507), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L479 - UNKNOWN_ACCESS_TYPE(508), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L480 - INVALID_GRANT(509), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L481 - CACHE_DICTIONARY_UPDATE_FAIL(510), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L482 - UNKNOWN_ROLE(511), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L483 - SET_NON_GRANTED_ROLE(512), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L484 - UNKNOWN_PART_TYPE(513), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L485 - ACCESS_STORAGE_FOR_INSERTION_NOT_FOUND(514), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L486 - INCORRECT_ACCESS_ENTITY_DEFINITION(515), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L487 - AUTHENTICATION_FAILED(516), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L488 - CANNOT_ASSIGN_ALTER(517), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L489 - CANNOT_COMMIT_OFFSET(518), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L490 - NO_REMOTE_SHARD_AVAILABLE(519), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L491 - CANNOT_DETACH_DICTIONARY_AS_TABLE(520), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L492 - ATOMIC_RENAME_FAIL(521), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L493 - UNKNOWN_ROW_POLICY(523), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L494 - ALTER_OF_COLUMN_IS_FORBIDDEN(524), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L495 - INCORRECT_DISK_INDEX(525), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L496 - NO_SUITABLE_FUNCTION_IMPLEMENTATION(527), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L497 - CASSANDRA_INTERNAL_ERROR(528), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L498 - NOT_A_LEADER(529), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L499 - CANNOT_CONNECT_RABBITMQ(530), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L500 - CANNOT_FSTAT(531), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L501 - LDAP_ERROR(532), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L502 - INCONSISTENT_RESERVATIONS(533), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L503 - NO_RESERVATIONS_PROVIDED(534), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L504 - UNKNOWN_RAID_TYPE(535), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L505 - CANNOT_RESTORE_FROM_FIELD_DUMP(536), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L506 - ILLEGAL_MYSQL_VARIABLE(537), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L507 - MYSQL_SYNTAX_ERROR(538), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L508 - CANNOT_BIND_RABBITMQ_EXCHANGE(539), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L509 - CANNOT_DECLARE_RABBITMQ_EXCHANGE(540), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L510 - CANNOT_CREATE_RABBITMQ_QUEUE_BINDING(541), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L511 - CANNOT_REMOVE_RABBITMQ_EXCHANGE(542), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L512 - UNKNOWN_MYSQL_DATATYPES_SUPPORT_LEVEL(543), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L513 - ROW_AND_ROWS_TOGETHER(544), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L514 - FIRST_AND_NEXT_TOGETHER(545), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L515 - NO_ROW_DELIMITER(546), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L516 - INVALID_RAID_TYPE(547), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L517 - UNKNOWN_VOLUME(548), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L518 - DATA_TYPE_CANNOT_BE_USED_IN_KEY(549), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L519 - CONDITIONAL_TREE_PARENT_NOT_FOUND(550), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L520 - ILLEGAL_PROJECTION_MANIPULATOR(551), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L521 - UNRECOGNIZED_ARGUMENTS(552), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L522 - LZMA_STREAM_ENCODER_FAILED(553), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L523 - LZMA_STREAM_DECODER_FAILED(554), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L524 - ROCKSDB_ERROR(555), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L525 - SYNC_MYSQL_USER_ACCESS_ERROR(556), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L526 - UNKNOWN_UNION(557), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L527 - EXPECTED_ALL_OR_DISTINCT(558), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L528 - INVALID_GRPC_QUERY_INFO(559), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L529 - ZSTD_ENCODER_FAILED(560), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L530 - ZSTD_DECODER_FAILED(561), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L531 - TLD_LIST_NOT_FOUND(562), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L532 - CANNOT_READ_MAP_FROM_TEXT(563), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L533 - INTERSERVER_SCHEME_DOESNT_MATCH(564), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L534 - TOO_MANY_PARTITIONS(565), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L535 - CANNOT_RMDIR(566), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L536 - DUPLICATED_PART_UUIDS(567), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L537 - RAFT_ERROR(568), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L538 - MULTIPLE_COLUMNS_SERIALIZED_TO_SAME_PROTOBUF_FIELD(569), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L539 - DATA_TYPE_INCOMPATIBLE_WITH_PROTOBUF_FIELD(570), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L540 - DATABASE_REPLICATION_FAILED(571), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L541 - TOO_MANY_QUERY_PLAN_OPTIMIZATIONS(572), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L542 - EPOLL_ERROR(573), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L543 - DISTRIBUTED_TOO_MANY_PENDING_BYTES(574), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L544 - UNKNOWN_SNAPSHOT(575), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L545 - KERBEROS_ERROR(576), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L546 - INVALID_SHARD_ID(577), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L547 - INVALID_FORMAT_INSERT_QUERY_WITH_DATA(578), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L548 - INCORRECT_PART_TYPE(579), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L549 - CANNOT_SET_ROUNDING_MODE(580), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L550 - TOO_LARGE_DISTRIBUTED_DEPTH(581), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L551 - NO_SUCH_PROJECTION_IN_TABLE(582), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L552 - ILLEGAL_PROJECTION(583), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L553 - PROJECTION_NOT_USED(584), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L554 - CANNOT_PARSE_YAML(585), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L555 - CANNOT_CREATE_FILE(586), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L556 - CONCURRENT_ACCESS_NOT_SUPPORTED(587), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L557 - DISTRIBUTED_BROKEN_BATCH_INFO(588), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L558 - DISTRIBUTED_BROKEN_BATCH_FILES(589), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L559 - CANNOT_SYSCONF(590), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L560 - SQLITE_ENGINE_ERROR(591), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L561 - DATA_ENCRYPTION_ERROR(592), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L562 - ZERO_COPY_REPLICATION_ERROR(593), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L563 - BZIP2_STREAM_DECODER_FAILED(594), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L564 - BZIP2_STREAM_ENCODER_FAILED(595), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L565 - INTERSECT_OR_EXCEPT_RESULT_STRUCTURES_MISMATCH(596), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L566 - NO_SUCH_ERROR_CODE(597), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L567 - BACKUP_ALREADY_EXISTS(598), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L568 - BACKUP_NOT_FOUND(599), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L569 - BACKUP_VERSION_NOT_SUPPORTED(600), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L570 - BACKUP_DAMAGED(601), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L571 - NO_BASE_BACKUP(602), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L572 - WRONG_BASE_BACKUP(603), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L573 - BACKUP_ENTRY_ALREADY_EXISTS(604), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L574 - BACKUP_ENTRY_NOT_FOUND(605), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L575 - BACKUP_IS_EMPTY(606), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L576 - BACKUP_ELEMENT_DUPLICATE(607), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L577 - CANNOT_RESTORE_TABLE(608), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L578 - FUNCTION_ALREADY_EXISTS(609), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L579 - CANNOT_DROP_SYSTEM_FUNCTION(610), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L580 - CANNOT_CREATE_RECURSIVE_FUNCTION(611), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L581 - OBJECT_ALREADY_STORED_ON_DISK(612), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L582 - OBJECT_WAS_NOT_STORED_ON_DISK(613), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L583 - POSTGRESQL_CONNECTION_FAILURE(614), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L584 - CANNOT_ADVISE(615), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L585 - UNKNOWN_READ_METHOD(616), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L586 - LZ4_ENCODER_FAILED(617), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L587 - LZ4_DECODER_FAILED(618), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L588 - POSTGRESQL_REPLICATION_INTERNAL_ERROR(619), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L589 - QUERY_NOT_ALLOWED(620), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L590 - KEEPER_EXCEPTION(999), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L592 - POCO_EXCEPTION(1000), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L593 - STD_EXCEPTION(1001), // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L594 - UNKNOWN_EXCEPTION(1002); // https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/ErrorCodes.cpp#L595 - - private static final Map byCodes; - - static { - Map map = new HashMap<>(); - for (ClickHouseErrorCode errorCode : values()) { - map.put(errorCode.code, errorCode); - } - byCodes = Collections.unmodifiableMap(map); - } - - public final Integer code; - - ClickHouseErrorCode(Integer code) { - this.code = code; - } - - public static ClickHouseErrorCode fromCode(Integer code) { - return fromCodeOrDefault(code, null); - } - - public static ClickHouseErrorCode fromCodeOrDefault(Integer code, ClickHouseErrorCode defaultErrorCode) { - return byCodes.getOrDefault(code, defaultErrorCode); - } - - @Override - public String toString() { - return name() + " (code " + code + ')'; - } -} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/exception/ClickHouseException.java b/clickhouse-client/src/main/java/com/clickhouse/client/exception/ClickHouseException.java deleted file mode 100644 index a471b258c..000000000 --- a/clickhouse-client/src/main/java/com/clickhouse/client/exception/ClickHouseException.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.clickhouse.client.exception; - -import com.clickhouse.client.ClickHouseNode; - -/** - * Exception thrown from ClickHouse server. - */ -public class ClickHouseException extends Exception { - /** - * Generated ID. - */ - private static final long serialVersionUID = -2417038200885554382L; - - private final int errorCode; - - private static String buildErrorMessage(ClickHouseErrorCode error, ClickHouseNode server, Throwable cause) { - if (error == null) { - error = ClickHouseErrorCode.UNKNOWN_EXCEPTION; - } - - return buildErrorMessage(error.name(), error.code, server, cause); - } - - private static String buildErrorMessage(String message, int code, ClickHouseNode server, Throwable cause) { - StringBuilder builder = new StringBuilder(); - - builder.append("ClickHouse exception, "); - if (message != null) { - builder.append(" message: ").append(message); - } else { - builder.append(" code: ").append(code); - } - - builder.append(", server: ").append(server).append(';'); - - if (cause != null) { - builder.append(' ').append(cause.getMessage()); - } - - return builder.toString(); - } - - public ClickHouseException(int code, Throwable cause, ClickHouseNode server) { - super(buildErrorMessage( - ClickHouseErrorCode.fromCodeOrDefault(code, ClickHouseErrorCode.UNKNOWN_EXCEPTION).name(), code, server, - cause), cause); - - errorCode = code; - } - - public ClickHouseException(int code, String message, Throwable cause, ClickHouseNode server) { - super(buildErrorMessage( - message != null ? message - : ClickHouseErrorCode.fromCodeOrDefault(code, ClickHouseErrorCode.UNKNOWN_EXCEPTION).name(), - code, server, cause), cause); - - errorCode = code; - } - - public ClickHouseException(int code, String message, Throwable cause) { - super(buildErrorMessage( - message != null ? message - : ClickHouseErrorCode.fromCodeOrDefault(code, ClickHouseErrorCode.UNKNOWN_EXCEPTION).name(), - code, null, cause), cause); - - errorCode = code; - } - - public ClickHouseException(ClickHouseErrorCode error, Throwable cause, ClickHouseNode server) { - super(buildErrorMessage(error, server, cause), cause); - - errorCode = error != null ? error.code : ClickHouseErrorCode.UNKNOWN_EXCEPTION.code; - } - - public int getErrorCode() { - return errorCode; - } -} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/exception/ClickHouseExceptionSpecifier.java b/clickhouse-client/src/main/java/com/clickhouse/client/exception/ClickHouseExceptionSpecifier.java deleted file mode 100644 index 78f0f4eab..000000000 --- a/clickhouse-client/src/main/java/com/clickhouse/client/exception/ClickHouseExceptionSpecifier.java +++ /dev/null @@ -1,112 +0,0 @@ -package com.clickhouse.client.exception; - -import java.net.ConnectException; -import java.net.SocketTimeoutException; -import java.util.concurrent.ExecutionException; - -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseNode; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; - -/** - * Specify ClickHouse exception to ClickHouseException and fill it with a vendor - * code. - */ -public final class ClickHouseExceptionSpecifier { - - private static final Logger log = LoggerFactory.getLogger(ClickHouseExceptionSpecifier.class); - - private ClickHouseExceptionSpecifier() { - } - - /** - * Handles ExecutionException. - * - * @param e ExecutionException - * @param server server - */ - public static ClickHouseException handle(ExecutionException e, ClickHouseNode server) { - Throwable cause = e.getCause(); - if (cause == null) { - cause = e; - } - return ClickHouseExceptionSpecifier.specify(cause, server); - } - - public static ClickHouseException specify(Throwable cause, ClickHouseNode server) { - return specify(cause != null ? cause.getMessage() : null, cause, server); - } - - public static ClickHouseException specify(String clickHouseMessage, ClickHouseNode server) { - return specify(clickHouseMessage, null, server); - } - - public static ClickHouseException specify(String clickHouseMessage) { - return specify(clickHouseMessage, null); - } - - /** - * Here we expect the ClickHouse error message to be of the following format: - * "Code: 10, e.displayText() = DB::Exception: ...". - */ - private static ClickHouseException specify(String clickHouseMessage, Throwable cause, ClickHouseNode server) { - if (ClickHouseChecker.isNullOrEmpty(clickHouseMessage) && cause != null) { - return getException(cause, server); - } - - try { - int code; - if (clickHouseMessage.startsWith("Poco::Exception. Code: 1000, ")) { - code = 1000; - } else { - // Code: 175, e.displayText() = DB::Exception: - code = getErrorCode(clickHouseMessage); - } - // ошибку в изначальном виде все-таки укажем - Throwable messageHolder = cause != null ? cause : new Throwable(clickHouseMessage); - if (code == -1) { - return getException(messageHolder, server); - } - - return new ClickHouseException(code, messageHolder, server); - } catch (Exception e) { - log.error( - "Unsupported ClickHouse error format, please fix ClickHouseExceptionSpecifier, message: {}, error: {}", - clickHouseMessage, e.getMessage()); - return new ClickHouseUnknownException(clickHouseMessage, cause, server); - } - } - - private static int getErrorCode(String errorMessage) { - int startIndex = errorMessage.indexOf(' '); - int endIndex = startIndex == -1 ? -1 : errorMessage.indexOf(',', startIndex); - - if (startIndex == -1 || endIndex == -1) { - return -1; - } - - try { - return Integer.parseInt(errorMessage.substring(startIndex + 1, endIndex)); - } catch (NumberFormatException e) { - return -1; - } - } - - private static ClickHouseException getException(Throwable cause, ClickHouseNode server) { - if (cause instanceof SocketTimeoutException) - // if we've got SocketTimeoutException, we'll say that the query is not good. - // This is not the same as SOCKET_TIMEOUT of clickhouse - // but it actually could be a failing ClickHouse - { - return new ClickHouseException(ClickHouseErrorCode.TIMEOUT_EXCEEDED.code, cause, server); - } else if (cause instanceof ConnectException) - // couldn't connect to ClickHouse during connectTimeout - { - return new ClickHouseException(ClickHouseErrorCode.NETWORK_ERROR.code, cause, server); - } else { - return new ClickHouseUnknownException(cause, server); - } - } - -} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/exception/ClickHouseUnknownException.java b/clickhouse-client/src/main/java/com/clickhouse/client/exception/ClickHouseUnknownException.java deleted file mode 100644 index cab313a61..000000000 --- a/clickhouse-client/src/main/java/com/clickhouse/client/exception/ClickHouseUnknownException.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.clickhouse.client.exception; - -import com.clickhouse.client.ClickHouseNode; - -public class ClickHouseUnknownException extends ClickHouseException { - /** - * Generated ID. - */ - private static final long serialVersionUID = -1724790228244438601L; - - public ClickHouseUnknownException(Throwable cause, ClickHouseNode server) { - super(ClickHouseErrorCode.UNKNOWN_EXCEPTION.code, cause, server); - } - - public ClickHouseUnknownException(String message, Throwable cause, ClickHouseNode server) { - super(ClickHouseErrorCode.UNKNOWN_EXCEPTION.code, message, cause, server); - } - - public ClickHouseUnknownException(String message, Throwable cause) { - super(ClickHouseErrorCode.UNKNOWN_EXCEPTION.code, message, cause); - } - - public ClickHouseUnknownException(Integer code, Throwable cause, ClickHouseNode server) { - super(code, cause, server); - } -} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/exception/package-info.java b/clickhouse-client/src/main/java/com/clickhouse/client/exception/package-info.java deleted file mode 100644 index e1cb83105..000000000 --- a/clickhouse-client/src/main/java/com/clickhouse/client/exception/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Provides shared exception classes. - */ -package com.clickhouse.client.exception; diff --git a/clickhouse-client/src/main/java9/module-info.java b/clickhouse-client/src/main/java9/module-info.java index 02958f9fc..6ec52aaf9 100644 --- a/clickhouse-client/src/main/java9/module-info.java +++ b/clickhouse-client/src/main/java9/module-info.java @@ -5,19 +5,8 @@ exports com.clickhouse.client; exports com.clickhouse.client.config; exports com.clickhouse.client.data; - exports com.clickhouse.client.exception; - - exports com.clickhouse.client.logging to - com.clickhouse.client.http, - com.clickhouse.client.grpc, - com.clickhouse.client.mysql, - com.clickhouse.client.postgresql, - // native is a reserved keyword :< - com.clickhouse.client.tcp, - com.clickhouse.jdbc, - ru.yandex.clickhouse; - - requires java.base; + exports com.clickhouse.client.data.array; + exports com.clickhouse.client.logging; requires static java.logging; requires static com.google.gson; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/AbstractClientTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/AbstractClientTest.java new file mode 100644 index 000000000..8a4f34f28 --- /dev/null +++ b/clickhouse-client/src/test/java/com/clickhouse/client/AbstractClientTest.java @@ -0,0 +1,174 @@ +package com.clickhouse.client; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class AbstractClientTest { + static class SimpleClient extends AbstractClient { + @Override + public CompletableFuture execute(ClickHouseRequest request) { + return null; + } + + @Override + protected Object[] newConnection(Object[] connection, ClickHouseNode server, ClickHouseRequest request) { + if (connection != null) { + closeConnection(connection, false); + } + + return new Object[] { request.getConfig(), server }; + } + + @Override + protected void closeConnection(Object[] connection, boolean force) { + connection[0] = null; + connection[1] = null; + } + } + + @Test(groups = { "unit" }) + public void testClose() { + SimpleClient sc = new SimpleClient(); + Assert.assertFalse(sc.isInitialized()); + sc.close(); + Assert.assertFalse(sc.isInitialized()); + + sc.init(new ClickHouseConfig()); + Assert.assertNotNull(sc.getExecutor()); + Assert.assertTrue(sc.isInitialized()); + Assert.assertNotNull(sc.getConfig()); + Assert.assertNull(sc.getServer()); + sc.close(); + Assert.assertFalse(sc.isInitialized()); + Assert.assertThrows(IllegalStateException.class, () -> sc.getConfig()); + Assert.assertThrows(IllegalStateException.class, () -> sc.getExecutor()); + Assert.assertThrows(IllegalStateException.class, () -> sc.getServer()); + + SimpleClient client = new SimpleClient(); + client.init(new ClickHouseConfig()); + ClickHouseRequest req = client.connect(ClickHouseNode.builder().build()); + ClickHouseConfig config = new ClickHouseConfig(); + sc.init(config); + Assert.assertNotNull(sc.getExecutor()); + Assert.assertTrue(sc.isInitialized()); + Assert.assertNotNull(sc.getConfig()); + Assert.assertNull(sc.getServer()); + Assert.assertEquals(sc.getConnection(req), new Object[] { req.getConfig(), req.getServer() }); + sc.close(); + Assert.assertFalse(sc.isInitialized()); + Assert.assertThrows(IllegalStateException.class, () -> sc.getConfig()); + Assert.assertThrows(IllegalStateException.class, () -> sc.getConnection(req)); + Assert.assertThrows(IllegalStateException.class, () -> sc.getExecutor()); + Assert.assertThrows(IllegalStateException.class, () -> sc.getServer()); + } + + @Test(groups = { "unit" }) + public void testCloseRunningClient() throws InterruptedException { + SimpleClient client = new SimpleClient(); + client.init(new ClickHouseConfig()); + ClickHouseRequest req = client.connect(ClickHouseNode.builder().build()); + + CountDownLatch latch = new CountDownLatch(1); + new Thread(() -> { + try { + Object[] conn = client.getConnection(req); + Thread.sleep(1000L); + client.close(); + latch.countDown(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }).start(); + Assert.assertTrue(client.isInitialized()); + Assert.assertTrue(latch.await(5000L, TimeUnit.MILLISECONDS)); + Assert.assertFalse(client.isInitialized()); + } + + @Test(groups = { "unit" }) + public void testGetAndCloseConnection() { + SimpleClient client = new SimpleClient(); + client.init(new ClickHouseConfig()); + ClickHouseRequest req = client.connect(ClickHouseNode.builder().build()); + + SimpleClient sc = new SimpleClient(); + sc.init(new ClickHouseConfig()); + Assert.assertEquals(sc.getConnection(req), new Object[] { req.getConfig(), req.getServer() }); + + req = client.connect(ClickHouseNode.of("127.0.0.1", ClickHouseProtocol.GRPC, 9100, "test")); + Object[] conn = sc.getConnection(req); + Assert.assertEquals(conn, new Object[] { req.getConfig(), req.getServer() }); + sc.close(); + Assert.assertNull(conn[0]); + Assert.assertNull(conn[1]); + } + + @Test(groups = { "unit" }) + public void testInit() { + SimpleClient client = new SimpleClient(); + client.init(new ClickHouseConfig()); + ClickHouseRequest req = client.connect(ClickHouseNode.builder().build()); + + SimpleClient sc = new SimpleClient(); + Assert.assertFalse(sc.isInitialized()); + Assert.assertThrows(IllegalStateException.class, () -> sc.connect(ClickHouseNode.builder().build())); + Assert.assertThrows(IllegalStateException.class, () -> sc.getConfig()); + Assert.assertThrows(IllegalStateException.class, () -> sc.getConnection(req)); + Assert.assertThrows(IllegalStateException.class, () -> sc.getExecutor()); + Assert.assertThrows(IllegalStateException.class, () -> sc.getServer()); + + ClickHouseConfig config = new ClickHouseConfig(); + sc.init(config); + Assert.assertEquals(sc.getExecutor(), ClickHouseClient.getExecutorService()); + Assert.assertTrue(sc.isInitialized()); + Assert.assertTrue(sc.getConfig() == config); + Assert.assertNull(sc.getServer()); + Assert.assertEquals(sc.getConnection(req), new Object[] { req.getConfig(), req.getServer() }); + Assert.assertEquals(sc.getServer(), req.getServer()); + + ClickHouseConfig newConfig = new ClickHouseConfig(); + sc.init(newConfig); + Assert.assertEquals(sc.getExecutor(), ClickHouseClient.getExecutorService()); + Assert.assertTrue(sc.isInitialized()); + Assert.assertTrue(sc.getConfig() != config); + Assert.assertEquals(sc.getConnection(req), new Object[] { req.getConfig(), req.getServer() }); + Assert.assertEquals(sc.getServer(), req.getServer()); + } + + @Test(groups = { "unit" }) + public void testSwitchNode() throws InterruptedException { + ClickHouseConfig config = new ClickHouseConfig(); + SimpleClient client = new SimpleClient(); + client.init(config); + ClickHouseRequest req1 = client.connect(ClickHouseNode.builder().build()); + ClickHouseRequest req2 = client + .connect(ClickHouseNode.of("127.0.0.1", ClickHouseProtocol.GRPC, 9100, "test")); + + Object[] conn1 = client.getConnection(req1); + CountDownLatch latch = new CountDownLatch(1); + new Thread(() -> { + try { + Thread.sleep(1000L); + Object[] conn2 = client.getConnection(req2); + latch.countDown(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }).start(); + Assert.assertEquals(conn1, new Object[] { req1.getConfig(), req1.getServer() }); + Assert.assertTrue(latch.await(5000L, TimeUnit.MILLISECONDS)); + Assert.assertTrue(client.isInitialized()); + Assert.assertNull(conn1[0]); + Assert.assertNull(conn1[1]); + Object[] conn2 = client.getConnection(req2); + Assert.assertTrue(conn1 != conn2); + Assert.assertEquals(conn2, new Object[] { req2.getConfig(), req2.getServer() }); + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/BaseIntegrationTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/BaseIntegrationTest.java index c273ee536..e2fff9cb2 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/BaseIntegrationTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/BaseIntegrationTest.java @@ -1,5 +1,9 @@ package com.clickhouse.client; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Locale; + import org.testng.annotations.BeforeTest; /** @@ -22,4 +26,15 @@ protected ClickHouseNode getServer(ClickHouseProtocol protocol, ClickHouseNode b protected ClickHouseNode getServer(ClickHouseProtocol protocol, int port) { return ClickHouseServerForTest.getClickHouseNode(protocol, port); } + + protected String getIpAddress(ClickHouseNode server) { + String ipAddress = server.getHost(); + try { + ipAddress = InetAddress.getByName(ipAddress).getHostAddress(); + } catch (UnknownHostException e) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Not able to resolve %s to get its IP address", server.getHost()), e); + } + return ipAddress; + } } diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClientBuilderTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClientBuilderTest.java index 96980fb03..69eb6f9f4 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClientBuilderTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClientBuilderTest.java @@ -24,7 +24,7 @@ public void testBuildConfig() { Assert.assertEquals(builder.getConfig(), config); String clientName = "test client"; - builder.addOption(ClickHouseClientOption.CLIENT_NAME, clientName); + builder.option(ClickHouseClientOption.CLIENT_NAME, clientName); Assert.assertNotEquals(builder.getConfig(), config); config = builder.getConfig(); Assert.assertEquals(config.getClientName(), clientName); diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClusterTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClusterTest.java index bda5c6f45..ae478f332 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClusterTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClusterTest.java @@ -104,7 +104,7 @@ public void testCheck(ClickHouseNodeSelector nodeSelector) { public void testProbe() { // FIXME does not support ClickHouseProtocol.POSTGRESQL for now ClickHouseProtocol[] protocols = new ClickHouseProtocol[] { ClickHouseProtocol.GRPC, ClickHouseProtocol.HTTP, - ClickHouseProtocol.MYSQL, ClickHouseProtocol.NATIVE }; + ClickHouseProtocol.MYSQL, ClickHouseProtocol.TCP }; for (ClickHouseProtocol p : protocols) { ClickHouseNode node = getServer(ClickHouseProtocol.ANY, p.getDefaultPort()); diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseColumnTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseColumnTest.java index 3ccc288af..105b4806e 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseColumnTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseColumnTest.java @@ -1,5 +1,6 @@ package com.clickhouse.client; +import java.util.Collections; import java.util.LinkedList; import java.util.List; import org.testng.Assert; @@ -107,4 +108,44 @@ public void testParse() throws Exception { list = ClickHouseColumn.parse("a String default 'cc', b String null"); Assert.assertEquals(list.size(), 2); } + + @Test(groups = { "unit" }) + public void testAggregationFunction() throws Exception { + ClickHouseColumn column = ClickHouseColumn.of("aggFunc", "AggregateFunction(groupBitmap, UInt32)"); + Assert.assertTrue(column.isAggregateFunction()); + Assert.assertEquals(column.getDataType(), ClickHouseDataType.AggregateFunction); + Assert.assertEquals(column.getAggregateFunction(), ClickHouseAggregateFunction.groupBitmap); + Assert.assertEquals(column.getFunction(), "groupBitmap"); + Assert.assertEquals(column.getNestedColumns(), Collections.singletonList(ClickHouseColumn.of("", "UInt32"))); + + column = ClickHouseColumn.of("aggFunc", "AggregateFunction(quantiles(0.5, 0.9), Nullable(UInt64))"); + Assert.assertTrue(column.isAggregateFunction()); + Assert.assertEquals(column.getDataType(), ClickHouseDataType.AggregateFunction); + Assert.assertEquals(column.getAggregateFunction(), ClickHouseAggregateFunction.quantiles); + Assert.assertEquals(column.getFunction(), "quantiles(0.5,0.9)"); + Assert.assertEquals(column.getNestedColumns(), + Collections.singletonList(ClickHouseColumn.of("", "Nullable(UInt64)"))); + } + + @Test(groups = { "unit" }) + public void testArray() throws Exception { + ClickHouseColumn column = ClickHouseColumn.of("arr", + "Array(Array(Array(Array(Array(Map(LowCardinality(String), Tuple(Array(UInt8),LowCardinality(String))))))))"); + Assert.assertTrue(column.isArray()); + Assert.assertEquals(column.getDataType(), ClickHouseDataType.Array); + Assert.assertEquals(column.getArrayNestedLevel(), 5); + Assert.assertEquals(column.getArrayBaseColumn().getOriginalTypeName(), + "Map(LowCardinality(String), Tuple(Array(UInt8),LowCardinality(String)))"); + Assert.assertFalse(column.getArrayBaseColumn().isArray()); + + Assert.assertEquals(column.getArrayBaseColumn().getArrayNestedLevel(), 0); + Assert.assertEquals(column.getArrayBaseColumn().getArrayBaseColumn(), null); + + ClickHouseColumn c = ClickHouseColumn.of("arr", "Array(LowCardinality(Nullable(String)))"); + Assert.assertTrue(c.isArray()); + Assert.assertEquals(c.getDataType(), ClickHouseDataType.Array); + Assert.assertEquals(c.getArrayNestedLevel(), 1); + Assert.assertEquals(c.getArrayBaseColumn().getOriginalTypeName(), "LowCardinality(Nullable(String))"); + Assert.assertFalse(c.getArrayBaseColumn().isArray()); + } } diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java index 54e983c1a..232d9eea3 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java @@ -1,75 +1,75 @@ package com.clickhouse.client; +import java.io.Serializable; import java.util.HashMap; import java.util.Map; import org.testng.Assert; import org.testng.annotations.Test; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseConfigOption; +import com.clickhouse.client.config.ClickHouseOption; import com.clickhouse.client.config.ClickHouseDefaults; public class ClickHouseConfigTest { - @Test(groups = { "unit" }) - public void testDefaultValues() { - ClickHouseConfig config = new ClickHouseConfig(null, null, null, null, null); - Assert.assertEquals(config.getClientName(), - ClickHouseClientOption.CLIENT_NAME.getEffectiveDefaultValue()); - Assert.assertEquals(config.getDatabase(), ClickHouseDefaults.DATABASE.getEffectiveDefaultValue()); + @Test(groups = { "unit" }) + public void testDefaultValues() { + ClickHouseConfig config = new ClickHouseConfig(null, null, null, null, null); + Assert.assertEquals(config.getClientName(), ClickHouseClientOption.CLIENT_NAME.getEffectiveDefaultValue()); + Assert.assertEquals(config.getDatabase(), ClickHouseDefaults.DATABASE.getEffectiveDefaultValue()); - Assert.assertEquals(config.getOption(ClickHouseDefaults.CLUSTER), - ClickHouseDefaults.CLUSTER.getEffectiveDefaultValue()); - Assert.assertEquals(config.getOption(ClickHouseDefaults.HOST), - ClickHouseDefaults.HOST.getEffectiveDefaultValue()); - Assert.assertEquals(config.getOption(ClickHouseDefaults.PORT), - ClickHouseDefaults.PORT.getEffectiveDefaultValue()); - Assert.assertEquals(config.getOption(ClickHouseDefaults.WEIGHT), - ClickHouseDefaults.WEIGHT.getEffectiveDefaultValue()); - ClickHouseCredentials credentials = config.getDefaultCredentials(); - Assert.assertEquals(credentials.useAccessToken(), false); - Assert.assertEquals(credentials.getUserName(), ClickHouseDefaults.USER.getEffectiveDefaultValue()); - Assert.assertEquals(credentials.getPassword(), ClickHouseDefaults.PASSWORD.getEffectiveDefaultValue()); - Assert.assertEquals(config.getFormat().name(), ClickHouseDefaults.FORMAT.getEffectiveDefaultValue()); - Assert.assertFalse(config.getMetricRegistry().isPresent()); - } + Assert.assertEquals(config.getOption(ClickHouseDefaults.CLUSTER), + ClickHouseDefaults.CLUSTER.getEffectiveDefaultValue()); + Assert.assertEquals(config.getOption(ClickHouseDefaults.HOST), + ClickHouseDefaults.HOST.getEffectiveDefaultValue()); + Assert.assertEquals(config.getOption(ClickHouseDefaults.PORT), + ClickHouseDefaults.PORT.getEffectiveDefaultValue()); + Assert.assertEquals(config.getOption(ClickHouseDefaults.WEIGHT), + ClickHouseDefaults.WEIGHT.getEffectiveDefaultValue()); + ClickHouseCredentials credentials = config.getDefaultCredentials(); + Assert.assertEquals(credentials.useAccessToken(), false); + Assert.assertEquals(credentials.getUserName(), ClickHouseDefaults.USER.getEffectiveDefaultValue()); + Assert.assertEquals(credentials.getPassword(), ClickHouseDefaults.PASSWORD.getEffectiveDefaultValue()); + Assert.assertEquals(config.getFormat(), ClickHouseDefaults.FORMAT.getEffectiveDefaultValue()); + Assert.assertFalse(config.getMetricRegistry().isPresent()); + } - @Test(groups = { "unit" }) - public void testCustomValues() throws Exception { - String clientName = "test client"; - String cluster = "test cluster"; - String database = "test_database"; - String host = "test.host"; - Integer port = 12345; - Integer weight = -99; - String user = "sa"; - String password = "welcome"; + @Test(groups = { "unit" }) + public void testCustomValues() throws Exception { + String clientName = "test client"; + String cluster = "test cluster"; + String database = "test_database"; + String host = "test.host"; + Integer port = 12345; + Integer weight = -99; + String user = "sa"; + String password = "welcome"; - Map options = new HashMap<>(); - options.put(ClickHouseClientOption.CLIENT_NAME, clientName); - options.put(ClickHouseDefaults.CLUSTER, cluster); - options.put(ClickHouseClientOption.DATABASE, database); - options.put(ClickHouseDefaults.HOST, host); - options.put(ClickHouseDefaults.PORT, port); - options.put(ClickHouseDefaults.WEIGHT, weight); - options.put(ClickHouseDefaults.USER, "useless"); - options.put(ClickHouseDefaults.PASSWORD, "useless"); + Map options = new HashMap<>(); + options.put(ClickHouseClientOption.CLIENT_NAME, clientName); + options.put(ClickHouseDefaults.CLUSTER, cluster); + options.put(ClickHouseClientOption.DATABASE, database); + options.put(ClickHouseDefaults.HOST, host); + options.put(ClickHouseDefaults.PORT, port); + options.put(ClickHouseDefaults.WEIGHT, weight); + options.put(ClickHouseDefaults.USER, "useless"); + options.put(ClickHouseDefaults.PASSWORD, "useless"); - Object metricRegistry = new Object(); + Object metricRegistry = new Object(); - ClickHouseConfig config = new ClickHouseConfig(options, - ClickHouseCredentials.fromUserAndPassword(user, password), null, metricRegistry); - Assert.assertEquals(config.getClientName(), clientName); - Assert.assertEquals(config.getDatabase(), database); - Assert.assertEquals(config.getOption(ClickHouseDefaults.CLUSTER), cluster); - Assert.assertEquals(config.getOption(ClickHouseDefaults.HOST), host); - Assert.assertEquals(config.getOption(ClickHouseDefaults.PORT), port); - Assert.assertEquals(config.getOption(ClickHouseDefaults.WEIGHT), weight); + ClickHouseConfig config = new ClickHouseConfig(options, + ClickHouseCredentials.fromUserAndPassword(user, password), null, metricRegistry); + Assert.assertEquals(config.getClientName(), clientName); + Assert.assertEquals(config.getDatabase(), database); + Assert.assertEquals(config.getOption(ClickHouseDefaults.CLUSTER), cluster); + Assert.assertEquals(config.getOption(ClickHouseDefaults.HOST), host); + Assert.assertEquals(config.getOption(ClickHouseDefaults.PORT), port); + Assert.assertEquals(config.getOption(ClickHouseDefaults.WEIGHT), weight); - ClickHouseCredentials credentials = config.getDefaultCredentials(); - Assert.assertEquals(credentials.useAccessToken(), false); - Assert.assertEquals(credentials.getUserName(), user); - Assert.assertEquals(credentials.getPassword(), password); - Assert.assertEquals(config.getPreferredProtocols().size(), 0); - Assert.assertEquals(config.getPreferredTags().size(), 0); - Assert.assertEquals(config.getMetricRegistry().get(), metricRegistry); - } + ClickHouseCredentials credentials = config.getDefaultCredentials(); + Assert.assertEquals(credentials.useAccessToken(), false); + Assert.assertEquals(credentials.getUserName(), user); + Assert.assertEquals(credentials.getPassword(), password); + Assert.assertEquals(config.getPreferredProtocols().size(), 0); + Assert.assertEquals(config.getPreferredTags().size(), 0); + Assert.assertEquals(config.getMetricRegistry().get(), metricRegistry); + } } diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseExceptionTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseExceptionTest.java new file mode 100644 index 000000000..ad41f1bec --- /dev/null +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseExceptionTest.java @@ -0,0 +1,109 @@ +package com.clickhouse.client; + +import java.util.concurrent.ExecutionException; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseExceptionTest { + @Test(groups = { "unit" }) + public void testConstructorWithCause() { + ClickHouseException e = new ClickHouseException(-1, (Throwable) null, null); + Assert.assertEquals(e.getErrorCode(), -1); + Assert.assertNull(e.getCause()); + Assert.assertEquals(e.getMessage(), "Unknown error -1"); + + ClickHouseNode server = ClickHouseNode.builder().build(); + e = new ClickHouseException(233, (Throwable) null, server); + Assert.assertEquals(e.getErrorCode(), 233); + Assert.assertNull(e.getCause()); + Assert.assertEquals(e.getMessage(), "Unknown error 233 on server " + server); + + Throwable cause = new IllegalArgumentException(); + e = new ClickHouseException(123, cause, server); + Assert.assertEquals(e.getErrorCode(), 123); + Assert.assertEquals(e.getCause(), cause); + Assert.assertEquals(e.getMessage(), "Unknown error 123 on server " + server); + + cause = new IllegalArgumentException("Some error"); + e = new ClickHouseException(111, cause, server); + Assert.assertEquals(e.getErrorCode(), 111); + Assert.assertEquals(e.getCause(), cause); + Assert.assertEquals(e.getMessage(), "Some error on server " + server); + } + + @Test(groups = { "unit" }) + public void testConstructorWithoutCause() { + ClickHouseException e = new ClickHouseException(-1, (String) null, null); + Assert.assertEquals(e.getErrorCode(), -1); + Assert.assertNull(e.getCause()); + Assert.assertEquals(e.getMessage(), "Unknown error -1"); + + ClickHouseNode server = ClickHouseNode.builder().build(); + e = new ClickHouseException(233, (String) null, server); + Assert.assertEquals(e.getErrorCode(), 233); + Assert.assertNull(e.getCause()); + Assert.assertEquals(e.getMessage(), "Unknown error 233 on server " + server); + + e = new ClickHouseException(123, "", server); + Assert.assertEquals(e.getErrorCode(), 123); + Assert.assertNull(e.getCause()); + Assert.assertEquals(e.getMessage(), "Unknown error 123 on server " + server); + + e = new ClickHouseException(111, "Some error", server); + Assert.assertEquals(e.getErrorCode(), 111); + Assert.assertNull(e.getCause()); + Assert.assertEquals(e.getMessage(), "Some error on server " + server); + } + + @Test(groups = { "unit" }) + public void testHandleException() { + ClickHouseNode server = ClickHouseNode.builder().build(); + Throwable cause = new RuntimeException(); + ClickHouseException e = ClickHouseException.of(cause, server); + Assert.assertEquals(e.getErrorCode(), ClickHouseException.ERROR_UNKNOWN); + Assert.assertEquals(e.getCause(), cause); + Assert.assertEquals(e.getMessage(), + "Unknown error " + ClickHouseException.ERROR_UNKNOWN + " on server " + server); + + e = ClickHouseException.of("Some error", server); + Assert.assertEquals(e.getErrorCode(), ClickHouseException.ERROR_UNKNOWN); + Assert.assertNull(e.getCause()); + Assert.assertEquals(e.getMessage(), "Some error on server " + server); + + Assert.assertEquals(e, ClickHouseException.of(e, server)); + + cause = new ExecutionException(null); + e = ClickHouseException.of(cause, server); + Assert.assertEquals(e.getErrorCode(), ClickHouseException.ERROR_UNKNOWN); + Assert.assertEquals(e.getCause(), cause); + Assert.assertEquals(e.getMessage(), + "Unknown error " + ClickHouseException.ERROR_UNKNOWN + " on server " + server); + + e = ClickHouseException.of((ExecutionException) null, server); + Assert.assertEquals(e.getErrorCode(), ClickHouseException.ERROR_UNKNOWN); + Assert.assertNull(e.getCause()); + Assert.assertEquals(e.getMessage(), + "Unknown error " + ClickHouseException.ERROR_UNKNOWN + " on server " + server); + + cause = new ExecutionException(new ClickHouseException(-100, (Throwable) null, server)); + e = ClickHouseException.of(cause, server); + Assert.assertEquals(e, cause.getCause()); + Assert.assertEquals(e.getErrorCode(), -100); + Assert.assertNull(e.getCause()); + Assert.assertEquals(e.getMessage(), "Unknown error -100 on server " + server); + + cause = new ExecutionException(new IllegalArgumentException()); + e = ClickHouseException.of(cause, server); + Assert.assertEquals(e.getErrorCode(), ClickHouseException.ERROR_UNKNOWN); + Assert.assertEquals(e.getCause(), cause.getCause()); + Assert.assertEquals(e.getMessage(), + "Unknown error " + ClickHouseException.ERROR_UNKNOWN + " on server " + server); + + cause = new ExecutionException(new IllegalArgumentException("Code: 12345. Something goes wrong...")); + e = ClickHouseException.of(cause, server); + Assert.assertEquals(e.getErrorCode(), 12345); + Assert.assertEquals(e.getCause(), cause.getCause()); + Assert.assertEquals(e.getMessage(), cause.getCause().getMessage() + " on server " + server); + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java index c6fea3f2d..aa325928f 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java @@ -4,15 +4,17 @@ import org.testng.annotations.Test; import java.util.Arrays; +import java.util.Collections; +import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseDefaults; public class ClickHouseNodeTest { private void checkDefaultValues(ClickHouseNode node) { Assert.assertNotNull(node); Assert.assertEquals(node.getCluster(), ClickHouseDefaults.CLUSTER.getEffectiveDefaultValue()); - Assert.assertEquals(node.getDatabase(), ClickHouseDefaults.DATABASE.getEffectiveDefaultValue()); - Assert.assertEquals(node.getProtocol().name(), ClickHouseDefaults.PROTOCOL.getEffectiveDefaultValue()); + Assert.assertEquals(node.getDatabase().orElse(null), null); + Assert.assertEquals(node.getProtocol(), ClickHouseDefaults.PROTOCOL.getEffectiveDefaultValue()); Assert.assertFalse(node.getCredentials().isPresent()); Assert.assertTrue(node.getTags().isEmpty()); Assert.assertNotNull(node.getAddress()); @@ -30,7 +32,7 @@ private void checkCustomValues(ClickHouseNode node, String cluster, String host, Assert.assertEquals(node.getPort(), port); Assert.assertEquals(node.getWeight(), weight); Assert.assertEquals(node.getProtocol(), protocol); - Assert.assertEquals(node.getDatabase(), database); + Assert.assertEquals(node.getDatabase().orElse(null), database); Assert.assertEquals(node.getCredentials().orElse(null), credentials); Assert.assertEquals(node.getTags().size(), tags.length); for (String t : tags) { @@ -84,7 +86,7 @@ public void testBuildWithNode() { public void testBuildInOneGo() { String host = "non-existing.host"; String database = "my_db"; - ClickHouseProtocol protocol = ClickHouseProtocol.NATIVE; + ClickHouseProtocol protocol = ClickHouseProtocol.TCP; int port = 19000; ClickHouseNode node = ClickHouseNode.of(host, protocol, port, database); checkCustomValues(node, (String) ClickHouseDefaults.CLUSTER.getEffectiveDefaultValue(), host, port, @@ -96,4 +98,24 @@ public void testBuildInOneGo() { (int) ClickHouseDefaults.WEIGHT.getEffectiveDefaultValue(), protocol, database, null, new String[] { "read-only", "primary" }); } + + @Test(groups = { "unit" }) + public void testDatabase() { + ClickHouseConfig config = new ClickHouseConfig(Collections.singletonMap(ClickHouseClientOption.DATABASE, "ttt"), + null, null, null); + ClickHouseNode node = ClickHouseNode.builder().build(); + Assert.assertEquals(node.hasPreferredDatabase(), false); + Assert.assertEquals(node.getDatabase().orElse(null), null); + Assert.assertEquals(node.getDatabase(config), config.getDatabase()); + + node = ClickHouseNode.builder().database("").build(); + Assert.assertEquals(node.hasPreferredDatabase(), false); + Assert.assertEquals(node.getDatabase().orElse(null), ""); + Assert.assertEquals(node.getDatabase(config), config.getDatabase()); + + node = ClickHouseNode.builder().database("123").build(); + Assert.assertEquals(node.hasPreferredDatabase(), true); + Assert.assertEquals(node.getDatabase().orElse(null), "123"); + Assert.assertEquals(node.getDatabase(config), "123"); + } } diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseProtocolTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseProtocolTest.java new file mode 100644 index 000000000..187734387 --- /dev/null +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseProtocolTest.java @@ -0,0 +1,24 @@ +package com.clickhouse.client; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseProtocolTest { + @Test(groups = { "unit" }) + public void testUriScheme() { + Assert.assertThrows(UnsupportedOperationException.class, + () -> ClickHouseProtocol.GRPC.getUriSchemes().add("a")); + Assert.assertThrows(UnsupportedOperationException.class, + () -> ClickHouseProtocol.HTTP.getUriSchemes().remove(0)); + + for (ClickHouseProtocol p : ClickHouseProtocol.values()) { + for (String s : p.getUriSchemes()) { + Assert.assertEquals(ClickHouseProtocol.fromUriScheme(s), p); + Assert.assertEquals(ClickHouseProtocol.fromUriScheme(s.toUpperCase()), p); + Assert.assertEquals(ClickHouseProtocol.fromUriScheme(s + " "), ClickHouseProtocol.ANY); + } + } + + Assert.assertEquals(ClickHouseProtocol.fromUriScheme("gRPC"), ClickHouseProtocol.GRPC); + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseRequestTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseRequestTest.java index 4961e0f95..169357a75 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseRequestTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseRequestTest.java @@ -2,9 +2,11 @@ import java.io.ByteArrayInputStream; import java.math.BigInteger; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.UUID; import org.testng.Assert; @@ -37,15 +39,17 @@ public void testBuild() { String sql = "select 1"; request.table(table); - // Assert.assertNotEquals(config, request.getConfig()); + Assert.assertEquals(config, request.getConfig()); Assert.assertNotEquals(stmts, request.getStatements()); Assert.assertEquals(request.getStatements().size(), 1); Assert.assertEquals(request.getStatements().get(0), "SELECT * FROM " + table); request.query(sql); + Assert.assertEquals(config, request.getConfig()); Assert.assertEquals(request.getStatements().get(0), sql); request.use(db); + Assert.assertNotEquals(config, request.getConfig()); // because new option being added Assert.assertEquals(request.getConfig().getDatabase(), db); Assert.assertEquals(request.getStatements().size(), 1); Assert.assertEquals(request.getStatements().get(0), sql); @@ -66,7 +70,8 @@ public void testBuild() { @Test(groups = { "unit" }) public void testCopy() { ClickHouseRequest request = ClickHouseClient.newInstance().connect(ClickHouseNode.builder().build()); - request.compression(ClickHouseCompression.LZ4); + request.compressServerResponse(true, ClickHouseCompression.BROTLI, 2); + request.decompressClientRequest(true, ClickHouseCompression.ZSTD, 5); request.external(ClickHouseExternalTable.builder().content(new ByteArrayInputStream(new byte[0])).build()); request.format(ClickHouseFormat.Avro); request.table("table1", "query_id1"); @@ -117,16 +122,35 @@ public void testCopy() { public void testFormat() { ClickHouseRequest request = ClickHouseClient.newInstance().connect(ClickHouseNode.builder().build()); Assert.assertEquals(request.getFormat(), - ClickHouseFormat.valueOf((String) ClickHouseDefaults.FORMAT.getEffectiveDefaultValue())); - Assert.assertThrows(IllegalArgumentException.class, () -> request.format(null)); - Assert.assertEquals(request.getFormat(), - ClickHouseFormat.valueOf((String) ClickHouseDefaults.FORMAT.getEffectiveDefaultValue())); + (ClickHouseFormat) ClickHouseDefaults.FORMAT.getEffectiveDefaultValue()); request.format(ClickHouseFormat.ArrowStream); Assert.assertEquals(request.getFormat(), ClickHouseFormat.ArrowStream); + request.format(null); + Assert.assertEquals(request.getFormat(), + (ClickHouseFormat) ClickHouseDefaults.FORMAT.getEffectiveDefaultValue()); request.format(ClickHouseFormat.Arrow); Assert.assertEquals(request.getFormat(), ClickHouseFormat.Arrow); } + @Test(groups = { "unit" }) + public void testOptions() { + ClickHouseRequest request = ClickHouseClient.newInstance().connect(ClickHouseNode.builder().build()); + + Assert.assertEquals(request.options, Collections.emptyMap()); + Properties props = new Properties(); + props.setProperty(ClickHouseClientOption.ASYNC.getKey(), "false"); + props.setProperty(ClickHouseClientOption.DATABASE.getKey(), "mydb"); + props.setProperty(ClickHouseClientOption.CLIENT_NAME.getKey(), "new"); + props.setProperty(ClickHouseClientOption.FORMAT.getKey(), "CapnProto"); + request.options(props); + + Assert.assertEquals(request.options.size(), 4); + Assert.assertEquals(request.options.get(ClickHouseClientOption.ASYNC), false); + Assert.assertEquals(request.options.get(ClickHouseClientOption.DATABASE), "mydb"); + Assert.assertEquals(request.options.get(ClickHouseClientOption.CLIENT_NAME), "new"); + Assert.assertEquals(request.options.get(ClickHouseClientOption.FORMAT), ClickHouseFormat.CapnProto); + } + @Test(groups = { "unit" }) public void testParams() { String sql = "select :one as one, :two as two, * from my_table where key=:key and arr[:idx] in numbers(:range)"; @@ -171,7 +195,8 @@ public void testParams() { @Test(groups = { "unit" }) public void testSeal() { ClickHouseRequest request = ClickHouseClient.newInstance().connect(ClickHouseNode.builder().build()); - request.compression(ClickHouseCompression.LZ4); + request.compressServerResponse(true, ClickHouseCompression.BROTLI, 2); + request.decompressClientRequest(true, ClickHouseCompression.ZSTD, 5); request.external(ClickHouseExternalTable.builder().content(new ByteArrayInputStream(new byte[0])).build()); request.format(ClickHouseFormat.Avro); request.table("table1", "query_id1"); diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java index 497a556c8..36dd7eaac 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java @@ -106,7 +106,7 @@ public class ClickHouseServerForTest { .withExposedPorts(ClickHouseProtocol.GRPC.getDefaultPort(), ClickHouseProtocol.HTTP.getDefaultPort(), ClickHouseProtocol.MYSQL.getDefaultPort(), - ClickHouseProtocol.NATIVE.getDefaultPort(), + ClickHouseProtocol.TCP.getDefaultPort(), ClickHouseProtocol.POSTGRESQL.getDefaultPort()) .withClasspathResourceMapping("containers/clickhouse-server/config.d", "/etc/clickhouse-server/config.d", BindMode.READ_ONLY) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseTestClient.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseTestClient.java index bc003df86..7484565b3 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseTestClient.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseTestClient.java @@ -1,7 +1,6 @@ package com.clickhouse.client; import java.util.concurrent.CompletableFuture; -import com.clickhouse.client.exception.ClickHouseException; public class ClickHouseTestClient implements ClickHouseClient { private ClickHouseConfig clientConfig; @@ -12,7 +11,7 @@ public boolean accept(ClickHouseProtocol protocol) { } @Override - public CompletableFuture execute(ClickHouseRequest request) throws ClickHouseException { + public CompletableFuture execute(ClickHouseRequest request) { return CompletableFuture.supplyAsync(() -> null); } diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java index 42463fd63..bd99ba504 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java @@ -1,6 +1,8 @@ package com.clickhouse.client; import java.io.FileNotFoundException; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -21,6 +23,40 @@ public void testEscape() { Assert.assertEquals(ClickHouseUtils.escape("\\'", '\''), "\\\\\\'"); } + @Test(groups = { "unit" }) + public void testGetKeyValuePair() { + Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(null), Collections.emptyMap()); + Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(""), Collections.emptyMap()); + Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(" "), Collections.emptyMap()); + Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("="), Collections.emptyMap()); + Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("=="), Collections.emptyMap()); + Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(","), Collections.emptyMap()); + Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(",,"), Collections.emptyMap()); + Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("=,"), Collections.emptyMap()); + Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(" =\r ,"), Collections.emptyMap()); + Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("a\\=='b c',"), Collections.singletonMap("a=", "'b c'")); + Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("User-Agent=New Client, X-Forward-For=1\\,2"), + new HashMap() { + { + put("User-Agent", "New Client"); + put("X-Forward-For", "1,2"); + } + }); + } + + @Test(groups = { "unit" }) + public void testGetLeadingComment() { + Assert.assertEquals(ClickHouseUtils.getLeadingComment(null), ""); + Assert.assertEquals(ClickHouseUtils.getLeadingComment(""), ""); + Assert.assertEquals(ClickHouseUtils.getLeadingComment(" "), ""); + Assert.assertEquals(ClickHouseUtils.getLeadingComment("a"), ""); + Assert.assertEquals(ClickHouseUtils.getLeadingComment("-- a \r\nselect 1"), "a"); + Assert.assertEquals(ClickHouseUtils.getLeadingComment(" -- a \r\n-- b\nselect 1"), "a"); + Assert.assertEquals(ClickHouseUtils.getLeadingComment("/* a */select 1"), "a"); + Assert.assertEquals(ClickHouseUtils.getLeadingComment(" /* a /* b */*/ /*-- b*/\nselect 1"), "a /* b */"); + Assert.assertEquals(ClickHouseUtils.getLeadingComment("select /* a */1"), ""); + } + @Test(groups = { "unit" }) public void testGetService() { ClickHouseClient client = null; @@ -132,15 +168,97 @@ public void testSkipMultipleLineComment() { } @Test(groups = { "unit" }) - public void testSkipContentsUntil() { - String args = "select 'a' as `--b`,1/*('1(/*'*/(*/ from number(10)"; + public void testSkipContentsUntilCharacters() { + String args = "select 'a' as `--b`,1/*('1(/*'*/(\0*/ \0from number(10)"; + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length()), args.lastIndexOf('\0') + 1); Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), '('), args.lastIndexOf('(') + 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), '(', 'n'), + args.lastIndexOf('n') + 1); args = "column1 AggregateFunction(quantiles(0.5, 0.9), UInt64),\ncolumn2 UInt8 not null"; + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length()), args.length()); Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), ','), args.lastIndexOf(',') + 1); Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), '@'), args.length()); } + @Test(groups = { "unit" }) + public void testSkipContentsUntilKeyword() { + String args = "select 'a' as `--b`,1/*('1(/*'*/(\0*/ \0from number(10)"; + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), (String) null, true), 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), (String) null, false), 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "", true), 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "", false), 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "SELECT", true), args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "SELECT", false), + args.indexOf(' ')); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "'", true), args.indexOf('a')); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "'", false), args.indexOf('a')); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "From", true), args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "From", false), + args.lastIndexOf("from") + 4); + + args = "with (SELECT 1 as a) abcb -- select\nselect abcd"; + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "SELECT", true), args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "SELECT", false), + args.lastIndexOf(' ')); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "abcd", true), args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "abcd", false), args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil("abcd", 0, args.length(), "abcd", true), 4); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil("abcd", 0, args.length(), "abcd", false), 4); + + args = "column1 AggregateFunction(quantiles(0.5, 0.9), UInt64),\ncolumn2 UInt64 not null"; + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "uint128", true), args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "uint128", false), args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "uint64", true), args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "uint64", false), + args.lastIndexOf("UInt64") + 6); + } + + @Test(groups = { "unit" }) + public void testSkipContentsUntilKeywords() { + String args = "select 1 Insert, 2 as into"; + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), (String[]) null, true), 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), (String[]) null, false), 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[0], true), 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[0], false), 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { null }, true), 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { null }, false), 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "" }, true), 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "" }, false), 1); + + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "1", "insert" }, true), + args.length()); + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "1", "insert" }, false), + args.indexOf(',')); + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { " ", "insert" }, true), + args.length()); + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { " ", "insert" }, false), + args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "1", null }, true), + args.indexOf('I')); + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "1", null }, false), + args.indexOf('I')); + + args = "insert Into db.table(c1, c2) select d2, d3 From input('d1 String, d2 UInt8, d3 Array(UInt16)')"; + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "insert", "into" }, true), + args.length()); + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "insert", "into" }, false), + args.indexOf('d') - 1); + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "from", "input" }, true), + args.length()); + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "from", "input" }, false), + args.indexOf('\'') - 1); + } + @Test(groups = { "unit" }) public void testReadNameOrQuotedString() { String args = "123"; @@ -157,6 +275,15 @@ public void testReadNameOrQuotedString() { Assert.assertEquals(ClickHouseUtils.readNameOrQuotedString(args = " `1\"'2``3` ", 1, args.length(), builder), args.lastIndexOf('`') + 1); Assert.assertEquals(builder.toString(), "1\"'2`3"); + + builder.setLength(0); + Assert.assertEquals(ClickHouseUtils.readNameOrQuotedString(args = "input--", 0, args.length(), builder), + args.indexOf('-')); + Assert.assertEquals(builder.toString(), "input"); + builder.setLength(0); + Assert.assertEquals(ClickHouseUtils.readNameOrQuotedString(args = "input/*", 0, args.length(), builder), + args.indexOf('/')); + Assert.assertEquals(builder.toString(), "input"); } @Test(groups = { "unit" }) @@ -193,7 +320,17 @@ public void testReadParameters() { List params = new LinkedList<>(); Assert.assertEquals(ClickHouseUtils.readParameters(args, args.indexOf('('), args.length(), params), args.lastIndexOf(')') + 1); - Assert.assertEquals(params.size(), 2); + Assert.assertEquals(params, Arrays.asList("quantiles(0.5,'c \\'''([1],2) d',0.9)", "UInt64")); + + params.clear(); + args = " ('a'/* a*/, 1-- test\n, b)"; + Assert.assertEquals(ClickHouseUtils.readParameters(args, 0, args.length(), params), args.length()); + Assert.assertEquals(params, Arrays.asList("'a'", "1", "b")); + + params.clear(); + args = " a, b c"; + Assert.assertEquals(ClickHouseUtils.readParameters(args, 0, args.length(), params), args.length()); + Assert.assertEquals(params, Arrays.asList("a", "bc")); } @Test(groups = { "unit" }) @@ -201,4 +338,63 @@ public void testFileInputStream() { Assert.assertThrows(FileNotFoundException.class, () -> ClickHouseUtils.getFileInputStream(UUID.randomUUID().toString())); } + + @Test(groups = { "unit" }) + public void testParseJson() { + Assert.assertThrows(IllegalArgumentException.class, () -> ClickHouseUtils.parseJson(null)); + Assert.assertThrows(IllegalArgumentException.class, () -> ClickHouseUtils.parseJson("")); + + // constants + Assert.assertEquals(ClickHouseUtils.parseJson(" true"), Boolean.TRUE); + Assert.assertEquals(ClickHouseUtils.parseJson("false "), Boolean.FALSE); + Assert.assertEquals(ClickHouseUtils.parseJson(" null "), null); + + // numbers + Assert.assertEquals(ClickHouseUtils.parseJson("1"), 1); + Assert.assertEquals(ClickHouseUtils.parseJson("-1"), -1); + Assert.assertEquals(ClickHouseUtils.parseJson("1.1"), 1.1F); + Assert.assertEquals(ClickHouseUtils.parseJson("-1.1"), -1.1F); + Assert.assertEquals(ClickHouseUtils.parseJson("123456789.1"), 123456789.1D); + Assert.assertEquals(ClickHouseUtils.parseJson("-123456789.1"), -123456789.1D); + + // string + Assert.assertEquals(ClickHouseUtils.parseJson("\"\""), ""); + Assert.assertEquals(ClickHouseUtils.parseJson(" \" a \" "), " a "); + Assert.assertEquals(ClickHouseUtils.parseJson(" \"\\a\\/\" "), "a/"); + + // array + Assert.assertEquals(ClickHouseUtils.parseJson("[]"), new Object[0]); + Assert.assertEquals(ClickHouseUtils.parseJson(" [ ] "), new Object[0]); + Assert.assertEquals(ClickHouseUtils.parseJson("[1,2]"), new Object[] { 1, 2 }); + Assert.assertEquals(ClickHouseUtils.parseJson("[1, -2, true, 1.1, -2.1, \"ccc\"]"), + new Object[] { 1, -2, Boolean.TRUE, 1.1F, -2.1F, "ccc" }); + + // object + Assert.assertEquals(ClickHouseUtils.parseJson("{}"), Collections.emptyMap()); + Assert.assertEquals(ClickHouseUtils.parseJson(" { } "), Collections.emptyMap()); + + Map map = new HashMap<>(); + map.put("a", 1); + map.put("b", 2.2F); + map.put("c", null); + Assert.assertEquals(ClickHouseUtils.parseJson("{\"a\" : 1, \"c\": null, \"b\":2.2}"), map); + + map.clear(); + map.put("read_rows", "1"); + map.put("read_bytes", "12345678901"); + map.put("written_rows", "0.0"); + map.put("written_bytes", "0"); + map.put("total_rows_to_read", "233"); + Assert.assertEquals(ClickHouseUtils.parseJson( + "{\"read_rows\":\"1\",\"read_bytes\":\"12345678901\",\"written_rows\":\"0.0\",\"written_bytes\":\"0\",\"total_rows_to_read\":\"233\"}"), + map); + + // mixed + map.clear(); + map.put("a", 1); + map.put("b", 2.2F); + map.put("c", null); + Assert.assertEquals(ClickHouseUtils.parseJson("[null, 1, [2,3], {\"a\" : 1, \"c\": null, \"b\":2.2}]"), + new Object[] { null, 1, new Object[] { 2, 3 }, map }); + } } diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseValuesTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseValuesTest.java index 1c9606e82..410a966e8 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseValuesTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseValuesTest.java @@ -13,12 +13,64 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; +import java.util.Map; import java.util.UUID; import org.testng.Assert; import org.testng.annotations.Test; public class ClickHouseValuesTest extends BaseClickHouseValueTest { + @Test(groups = { "unit" }) + public void testCreateArray() { + Class[] primitiveTypes = new Class[] { boolean.class, byte.class, char.class, short.class, int.class, + long.class, float.class, double.class }; + Class[] wrapperTypes = new Class[] { Boolean.class, Byte.class, Character.class, Short.class, + Integer.class, Long.class, Float.class, Double.class }; + Class[] miscTypes = new Class[] { Object.class, Map.class, Collection.class, Class.class }; + for (Class c : primitiveTypes) { + Assert.assertEquals(ClickHouseValues.createObjectArray(c, 0, 1), ClickHouseValues.EMPTY_OBJECT_ARRAY); + } + for (Class c : wrapperTypes) { + Assert.assertEquals(ClickHouseValues.createObjectArray(c, 0, 1), ClickHouseValues.EMPTY_OBJECT_ARRAY); + } + for (Class c : miscTypes) { + Assert.assertEquals(ClickHouseValues.createObjectArray(c, 0, 1), ClickHouseValues.EMPTY_OBJECT_ARRAY); + } + + Object[] expectedValues = new Object[] { ClickHouseValues.EMPTY_BYTE_ARRAY, ClickHouseValues.EMPTY_BYTE_ARRAY, + ClickHouseValues.EMPTY_INT_ARRAY, ClickHouseValues.EMPTY_SHORT_ARRAY, ClickHouseValues.EMPTY_INT_ARRAY, + ClickHouseValues.EMPTY_LONG_ARRAY, ClickHouseValues.EMPTY_FLOAT_ARRAY, + ClickHouseValues.EMPTY_DOUBLE_ARRAY }; + int index = 0; + for (Class c : primitiveTypes) { + Assert.assertEquals(ClickHouseValues.createPrimitiveArray(c, 0, 1), expectedValues[index++]); + } + index = 0; + for (Class c : wrapperTypes) { + Assert.assertEquals(ClickHouseValues.createPrimitiveArray(c, 0, 1), expectedValues[index++]); + } + index = 0; + for (Class c : miscTypes) { + Assert.assertEquals(ClickHouseValues.createPrimitiveArray(c, 0, 1), ClickHouseValues.EMPTY_OBJECT_ARRAY); + } + + int[][] intArray = (int[][]) ClickHouseValues.createPrimitiveArray(int.class, 3, 2); + Assert.assertEquals(intArray, new int[][] { new int[0], new int[0], new int[0] }); + } + + @Test(groups = { "unit" }) + public void testNewArray() { + ClickHouseValue v = ClickHouseValues.newValue(ClickHouseColumn.of("a", "Array(UInt32)")); + Assert.assertEquals(v.asObject(), new long[0]); + v = ClickHouseValues.newValue(ClickHouseColumn.of("a", "Array(Array(UInt16))")); + Assert.assertEquals(v.asObject(), new int[0][]); + v = ClickHouseValues.newValue(ClickHouseColumn.of("a", "Array(Array(Array(Nullable(UInt8))))")); + Assert.assertEquals(v.asObject(), new short[0][][]); + v = ClickHouseValues.newValue(ClickHouseColumn.of("a", "Array(Array(Array(Array(LowCardinality(String)))))")); + Assert.assertEquals(v.asObject(), new String[0][][][]); + } + @Test(groups = { "unit" }) public void testConvertToDateTime() { Assert.assertEquals(ClickHouseValues.convertToDateTime(null), null); diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseVersionTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseVersionTest.java index c8718e498..a35d983f2 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseVersionTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseVersionTest.java @@ -130,6 +130,8 @@ public void testCheckRange() { Assert.assertTrue(ClickHouseVersion.parseVersion("21.3").check("(21.2,21.3]")); Assert.assertFalse(ClickHouseVersion.parseVersion("21.3").check("(21.3,21.4)")); Assert.assertTrue(ClickHouseVersion.parseVersion("21.3").check("[21.3,21.4)")); + + Assert.assertTrue(ClickHouseVersion.parseVersion("21.8.8.29").check("[18.16,)")); } @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/config/ClickHouseConfigOptionTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/config/ClickHouseConfigOptionTest.java index c40415554..6bfe97ef8 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/config/ClickHouseConfigOptionTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/config/ClickHouseConfigOptionTest.java @@ -1,12 +1,14 @@ package com.clickhouse.client.config; -import java.util.Optional; +import java.io.Serializable; import org.testng.Assert; import org.testng.annotations.Test; import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.client.ClickHouseFormat; public class ClickHouseConfigOptionTest { - static enum ClickHouseTestOption implements ClickHouseConfigOption { + static enum ClickHouseTestOption implements ClickHouseOption { STR("string_option", "string", "string option"), STR0("string_option0", "string0", "string option without environment variable support"), STR1("string_option0", "string1", "string option without environment variable and system property support"), @@ -18,13 +20,13 @@ static enum ClickHouseTestOption implements ClickHouseConfigOption { BOOL1("boolean_option1", false, "boolean option without environment variable and system property support"); private final String key; - private final Object defaultValue; - private final Class clazz; + private final Serializable defaultValue; + private final Class clazz; private final String description; - ClickHouseTestOption(String key, T defaultValue, String description) { + ClickHouseTestOption(String key, T defaultValue, String description) { this.key = ClickHouseChecker.nonNull(key, "key"); - this.defaultValue = Optional.of(defaultValue); + this.defaultValue = defaultValue; this.clazz = defaultValue.getClass(); this.description = ClickHouseChecker.nonNull(description, "description"); } @@ -35,7 +37,7 @@ public String getKey() { } @Override - public Object getDefaultValue() { + public Serializable getDefaultValue() { return defaultValue; } @@ -50,6 +52,33 @@ public String getDescription() { } } + @Test(groups = { "unit" }) + public void testFromString() { + Assert.assertThrows(IllegalArgumentException.class, + () -> ClickHouseOption.fromString(null, String.class)); + Assert.assertEquals(ClickHouseOption.fromString("", String.class), ""); + + Assert.assertEquals(ClickHouseOption.fromString("", Boolean.class), Boolean.FALSE); + Assert.assertEquals(ClickHouseOption.fromString("Yes", Boolean.class), Boolean.FALSE); + Assert.assertEquals(ClickHouseOption.fromString("1", Boolean.class), Boolean.TRUE); + Assert.assertEquals(ClickHouseOption.fromString("true", Boolean.class), Boolean.TRUE); + Assert.assertEquals(ClickHouseOption.fromString("True", Boolean.class), Boolean.TRUE); + + Assert.assertEquals(ClickHouseOption.fromString("", Integer.class), Integer.valueOf(0)); + Assert.assertEquals(ClickHouseOption.fromString("0", Integer.class), Integer.valueOf(0)); + Assert.assertThrows(IllegalArgumentException.class, + () -> ClickHouseOption.fromString(null, Integer.class)); + + Assert.assertEquals(ClickHouseOption.fromString("0.1", Float.class), Float.valueOf(0.1F)); + Assert.assertEquals(ClickHouseOption.fromString("NaN", Float.class), Float.valueOf(Float.NaN)); + + Assert.assertEquals(ClickHouseOption.fromString("Map", ClickHouseDataType.class), ClickHouseDataType.Map); + Assert.assertEquals(ClickHouseOption.fromString("RowBinary", ClickHouseFormat.class), + ClickHouseFormat.RowBinary); + Assert.assertThrows(IllegalArgumentException.class, + () -> ClickHouseOption.fromString("NonExistFormat", ClickHouseFormat.class)); + } + @Test(groups = { "unit" }) public void testGetEffectiveDefaultValue() { // environment variables are set in pom.xml diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/BinaryStreamUtilsTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/BinaryStreamUtilsTest.java index 4f3b87c42..dd9cfd924 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/BinaryStreamUtilsTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/BinaryStreamUtilsTest.java @@ -9,18 +9,33 @@ import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; -import java.util.Arrays; +import java.util.TimeZone; import java.util.UUID; +import com.clickhouse.client.ClickHouseDataType; + +import org.roaringbitmap.RoaringBitmap; +import org.roaringbitmap.buffer.ImmutableRoaringBitmap; +import org.roaringbitmap.buffer.MutableRoaringBitmap; +import org.roaringbitmap.longlong.Roaring64Bitmap; +import org.roaringbitmap.longlong.Roaring64NavigableMap; import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class BinaryStreamUtilsTest { + @DataProvider(name = "timeZoneProvider") + private Object[][] getTimeZones() { + return new Object[][] { new String[] { "Asia/Chongqing" }, new String[] { "America/Los_Angeles" }, + new String[] { "Europe/Moscow" }, new String[] { "Etc/UTC" }, new String[] { "Europe/Berlin" } }; + } + protected static byte[] generateBytes(int... ints) { byte[] bytes = new byte[ints.length]; for (int i = 0; i < ints.length; i++) { @@ -86,6 +101,46 @@ protected static byte[] getWrittenBytes(WriterFunction writter) throws IOExcepti } } + protected static int[] newBitmapValues(ClickHouseDataType t, int length) { + int[] array = new int[length]; + int base = (t.getByteLength() - 1) * 0xFF; + if (t.isSigned()) { + for (int i = 0; i < length; i++) { + if (i % 2 == 0) { + array[i / 2] = base + i; + } else { + array[length - i / 2 - 1] = (base + i) * -1; + } + } + } else { + for (int i = 0; i < length; i++) { + array[i] = base + i; + } + } + + return array; + } + + protected static long[] newBitmap64Values(ClickHouseDataType t, int length) { + long[] array = new long[length]; + long base = 0xFFFFFFFFL; + if (t.isSigned()) { + for (int i = 0; i < length; i++) { + if (i % 2 == 0) { + array[i / 2] = base + i; + } else { + array[length - i / 2 - 1] = (base + i) * -1; + } + } + } else { + for (int i = 0; i < length; i++) { + array[i] = base + i; + } + } + + return array; + } + @Test(groups = { "unit" }) public void testReverse() { byte[] bytes = null; @@ -133,6 +188,176 @@ public void testWriteByte() throws IOException { () -> getWrittenBytes(o -> BinaryStreamUtils.writeInt8(o, Byte.MIN_VALUE - 1))); } + @Test(groups = { "unit" }) + public void testWriteEmptyBitmap() throws IOException { + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeBitmap(o, ClickHouseBitmap.empty())), + generateBytes(0, 0)); + + for (ClickHouseDataType t : new ClickHouseDataType[] { ClickHouseDataType.Int8, ClickHouseDataType.UInt8, + ClickHouseDataType.Int16, ClickHouseDataType.UInt16, ClickHouseDataType.Int32, + ClickHouseDataType.UInt32, ClickHouseDataType.Int64, ClickHouseDataType.UInt64 }) { + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeBitmap(o, ClickHouseBitmap.empty(t))), + generateBytes(0, 0)); + } + } + + @Test(groups = { "unit" }) + public void testWriteBitmap32() throws IOException { + // cardinality <= 32 + for (ClickHouseDataType t : new ClickHouseDataType[] { ClickHouseDataType.Int8, ClickHouseDataType.UInt8, + ClickHouseDataType.Int16, ClickHouseDataType.UInt16, ClickHouseDataType.Int32, + ClickHouseDataType.UInt32 }) { + int[] values = newBitmapValues(t, 32); + byte[] expected = getWrittenBytes(o -> { + BinaryStreamUtils.writeInt8(o, 0); + BinaryStreamUtils.writeInt8(o, 0x20); + for (int v : values) { + int len = t.getByteLength(); + if (len == 1) { + BinaryStreamUtils.writeInt8(o, (byte) v); + } else if (len == 2) { + BinaryStreamUtils.writeInt16(o, (short) v); + } else { // 4 + BinaryStreamUtils.writeInt32(o, v); + } + } + }); + for (Object bitmap : new Object[] { RoaringBitmap.bitmapOf(values), MutableRoaringBitmap.bitmapOf(values), + ImmutableRoaringBitmap.bitmapOf(values) }) { + byte[] actual = getWrittenBytes( + o -> BinaryStreamUtils.writeBitmap(o, ClickHouseBitmap.wrap(bitmap, t))); + Assert.assertEquals(actual, expected); + } + } + + // cardinality > 32 + int i = 0; + for (ClickHouseDataType t : new ClickHouseDataType[] { ClickHouseDataType.Int8, ClickHouseDataType.UInt8, + ClickHouseDataType.Int16, ClickHouseDataType.UInt16, ClickHouseDataType.Int32, + ClickHouseDataType.UInt32 }) { + int[] values = newBitmapValues(t, 33); + byte[][] expected = new byte[][] { + generateBytes(0x01, 0x5A, 0x3A, 0x30, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, + 0xFF, 0xFF, 0x0F, 0x00, 0x18, 0x00, 0x00, 0x00, 0x3A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x04, 0x00, 0x06, 0x00, 0x08, 0x00, 0x0A, 0x00, 0x0C, 0x00, 0x0E, 0x00, 0x10, 0x00, + 0x12, 0x00, 0x14, 0x00, 0x16, 0x00, 0x18, 0x00, 0x1A, 0x00, 0x1C, 0x00, 0x1E, 0x00, 0x20, + 0x00, 0xE1, 0xFF, 0xE3, 0xFF, 0xE5, 0xFF, 0xE7, 0xFF, 0xE9, 0xFF, 0xEB, 0xFF, 0xED, 0xFF, + 0xEF, 0xFF, 0xF1, 0xFF, 0xF3, 0xFF, 0xF5, 0xFF, 0xF7, 0xFF, 0xF9, 0xFF, 0xFB, 0xFF, 0xFD, + 0xFF, 0xFF, 0xFF), + generateBytes(0x01, 0x52, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05, + 0x00, 0x06, 0x00, 0x07, 0x00, 0x08, 0x00, 0x09, 0x00, 0x0A, 0x00, 0x0B, 0x00, 0x0C, 0x00, + 0x0D, 0x00, 0x0E, 0x00, 0x0F, 0x00, 0x10, 0x00, 0x11, 0x00, 0x12, 0x00, 0x13, 0x00, 0x14, + 0x00, 0x15, 0x00, 0x16, 0x00, 0x17, 0x00, 0x18, 0x00, 0x19, 0x00, 0x1A, 0x00, 0x1B, 0x00, + 0x1C, 0x00, 0x1D, 0x00, 0x1E, 0x00, 0x1F, 0x00, 0x20, 0x00), + generateBytes(0x01, 0x5A, 0x3A, 0x30, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, + 0xFF, 0xFF, 0x0F, 0x00, 0x18, 0x00, 0x00, 0x00, 0x3A, 0x00, 0x00, 0x00, 0xFF, 0x00, 0x01, + 0x01, 0x03, 0x01, 0x05, 0x01, 0x07, 0x01, 0x09, 0x01, 0x0B, 0x01, 0x0D, 0x01, 0x0F, 0x01, + 0x11, 0x01, 0x13, 0x01, 0x15, 0x01, 0x17, 0x01, 0x19, 0x01, 0x1B, 0x01, 0x1D, 0x01, 0x1F, + 0x01, 0xE2, 0xFE, 0xE4, 0xFE, 0xE6, 0xFE, 0xE8, 0xFE, 0xEA, 0xFE, 0xEC, 0xFE, 0xEE, 0xFE, + 0xF0, 0xFE, 0xF2, 0xFE, 0xF4, 0xFE, 0xF6, 0xFE, 0xF8, 0xFE, 0xFA, 0xFE, 0xFC, 0xFE, 0xFE, + 0xFE, 0x00, 0xFF), + generateBytes(0x01, 0x52, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, + 0x10, 0x00, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x01, 0x01, 0x01, 0x02, 0x01, 0x03, 0x01, 0x04, + 0x01, 0x05, 0x01, 0x06, 0x01, 0x07, 0x01, 0x08, 0x01, 0x09, 0x01, 0x0A, 0x01, 0x0B, 0x01, + 0x0C, 0x01, 0x0D, 0x01, 0x0E, 0x01, 0x0F, 0x01, 0x10, 0x01, 0x11, 0x01, 0x12, 0x01, 0x13, + 0x01, 0x14, 0x01, 0x15, 0x01, 0x16, 0x01, 0x17, 0x01, 0x18, 0x01, 0x19, 0x01, 0x1A, 0x01, + 0x1B, 0x01, 0x1C, 0x01, 0x1D, 0x01, 0x1E, 0x01, 0x1F, 0x01), + generateBytes(0x01, 0x5A, 0x3A, 0x30, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, + 0xFF, 0xFF, 0x0F, 0x00, 0x18, 0x00, 0x00, 0x00, 0x3A, 0x00, 0x00, 0x00, 0xFD, 0x02, 0xFF, + 0x02, 0x01, 0x03, 0x03, 0x03, 0x05, 0x03, 0x07, 0x03, 0x09, 0x03, 0x0B, 0x03, 0x0D, 0x03, + 0x0F, 0x03, 0x11, 0x03, 0x13, 0x03, 0x15, 0x03, 0x17, 0x03, 0x19, 0x03, 0x1B, 0x03, 0x1D, + 0x03, 0xE4, 0xFC, 0xE6, 0xFC, 0xE8, 0xFC, 0xEA, 0xFC, 0xEC, 0xFC, 0xEE, 0xFC, 0xF0, 0xFC, + 0xF2, 0xFC, 0xF4, 0xFC, 0xF6, 0xFC, 0xF8, 0xFC, 0xFA, 0xFC, 0xFC, 0xFC, 0xFE, 0xFC, 0x00, + 0xFD, 0x02, 0xFD), + generateBytes(0x01, 0x52, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, + 0x10, 0x00, 0x00, 0x00, 0xFD, 0x02, 0xFE, 0x02, 0xFF, 0x02, 0x00, 0x03, 0x01, 0x03, 0x02, + 0x03, 0x03, 0x03, 0x04, 0x03, 0x05, 0x03, 0x06, 0x03, 0x07, 0x03, 0x08, 0x03, 0x09, 0x03, + 0x0A, 0x03, 0x0B, 0x03, 0x0C, 0x03, 0x0D, 0x03, 0x0E, 0x03, 0x0F, 0x03, 0x10, 0x03, 0x11, + 0x03, 0x12, 0x03, 0x13, 0x03, 0x14, 0x03, 0x15, 0x03, 0x16, 0x03, 0x17, 0x03, 0x18, 0x03, + 0x19, 0x03, 0x1A, 0x03, 0x1B, 0x03, 0x1C, 0x03, 0x1D, 0x03) }; + for (Object bitmap : new Object[] { RoaringBitmap.bitmapOf(values), MutableRoaringBitmap.bitmapOf(values), + ImmutableRoaringBitmap.bitmapOf(values) }) { + byte[] actual = getWrittenBytes( + o -> BinaryStreamUtils.writeBitmap(o, ClickHouseBitmap.wrap(bitmap, t))); + Assert.assertEquals(actual, expected[i]); + } + i++; + } + } + + @Test(groups = { "unit" }) + public void testWriteBitmap64() throws IOException { + // cardinality <= 32 + for (ClickHouseDataType t : new ClickHouseDataType[] { ClickHouseDataType.Int64, ClickHouseDataType.UInt64 }) { + long[] values = newBitmap64Values(t, 32); + byte[] expected = getWrittenBytes(o -> { + BinaryStreamUtils.writeInt8(o, 0); + BinaryStreamUtils.writeInt8(o, 0x20); + for (long v : values) { + BinaryStreamUtils.writeInt64(o, v); + } + }); + for (Object bitmap : new Object[] { Roaring64Bitmap.bitmapOf(values), + Roaring64NavigableMap.bitmapOf(values) }) { + byte[] actual = getWrittenBytes( + o -> BinaryStreamUtils.writeBitmap(o, ClickHouseBitmap.wrap(bitmap, t))); + Assert.assertEquals(actual, expected); + } + } + + // cardinality > 32 + // int i = 0; + // for (ClickHouseDataType t : new ClickHouseDataType[] { + // ClickHouseDataType.Int64, ClickHouseDataType.UInt64 }) { + // long[] values = newBitmap64Values(t, 33); + // byte[][] expected = new byte[][] { + // generateBytes(0x01, 0x9A, 0x01, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // 0x00, 0x00, 0x00, 0x00, + // 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, + // 0x10, 0x00, + // 0x00, 0x00, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, + // 0x00, 0x00, + // 0x00, 0x00, 0x00, 0x0F, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x03, 0x00, + // 0x05, 0x00, + // 0x07, 0x00, 0x09, 0x00, 0x0B, 0x00, 0x0D, 0x00, 0x0F, 0x00, 0x11, 0x00, 0x13, + // 0x00, 0x15, + // 0x00, 0x17, 0x00, 0x19, 0x00, 0x1B, 0x00, 0x1D, 0x00, 0x1F, 0x00, 0xFE, 0xFF, + // 0xFF, 0xFF, + // 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x0E, 0x00, 0x10, + // 0x00, 0x00, + // 0x00, 0xE2, 0xFF, 0xE4, 0xFF, 0xE6, 0xFF, 0xE8, 0xFF, 0xEA, 0xFF, 0xEC, 0xFF, + // 0xEE, 0xFF, + // 0xF0, 0xFF, 0xF2, 0xFF, 0xF4, 0xFF, 0xF6, 0xFF, 0xF8, 0xFF, 0xFA, 0xFF, 0xFC, + // 0xFF, 0xFE, + // 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + // 0x00, 0x00, + // 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00), + // generateBytes(0x01, 0x72, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // 0x00, 0x00, 0x00, 0x00, + // 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0x10, + // 0x00, 0x00, + // 0x00, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, + // 0x00, 0x00, + // 0x00, 0x00, 0x1F, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, + // 0x00, 0x03, + // 0x00, 0x04, 0x00, 0x05, 0x00, 0x06, 0x00, 0x07, 0x00, 0x08, 0x00, 0x09, 0x00, + // 0x0A, 0x00, + // 0x0B, 0x00, 0x0C, 0x00, 0x0D, 0x00, 0x0E, 0x00, 0x0F, 0x00, 0x10, 0x00, 0x11, + // 0x00, 0x12, + // 0x00, 0x13, 0x00, 0x14, 0x00, 0x15, 0x00, 0x16, 0x00, 0x17, 0x00, 0x18, 0x00, + // 0x19, 0x00, + // 0x1A, 0x00, 0x1B, 0x00, 0x1C, 0x00, 0x1D, 0x00, 0x1E, 0x00, 0x1F, 0x00) }; + // for (Object bitmap : new Object[] { Roaring64NavigableMap.bitmapOf(values) }) + // { + // byte[] actual = getWrittenBytes( + // o -> BinaryStreamUtils.writeBitmap(o, ClickHouseBitmap.wrap(bitmap, t))); + // Assert.assertEquals(actual, expected[i]); + // } + // i++; + // } + } + @Test(groups = { "unit" }) public void testReadUnsignedByte() throws IOException { for (int i = 0; i < 0xFF; i++) { @@ -529,188 +754,378 @@ public void testWriteUnsignedInt256() throws IOException { @Test(groups = { "unit" }) public void testReadDate() throws IOException { - Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(-1, 0)), LocalDate.ofEpochDay(255)); - Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(0, 0x80)), LocalDate.ofEpochDay(0x8000)); - Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(0, 0)), LocalDate.ofEpochDay(0)); - Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(1, 0)), LocalDate.ofEpochDay(1)); - Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(0xFF, 0x7F)), + Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(-1, 0), null), LocalDate.ofEpochDay(255)); + Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(0, 0x80), null), LocalDate.ofEpochDay(0x8000)); + Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(0, 0), null), LocalDate.ofEpochDay(0)); + Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(1, 0), null), LocalDate.ofEpochDay(1)); + Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(0xFF, 0x7F), null), LocalDate.ofEpochDay(Short.MAX_VALUE)); - Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(0xFF, 0xFF)), LocalDate.ofEpochDay(0xFFFF)); + Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(0xFF, 0xFF), null), LocalDate.ofEpochDay(0xFFFF)); - Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(0x9E, 0x49)), LocalDate.of(2021, 8, 7)); + Assert.assertEquals(BinaryStreamUtils.readDate(generateInput(0x9E, 0x49), null), LocalDate.of(2021, 8, 7)); } @Test(groups = { "unit" }) public void testWriteDate() throws IOException { - Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.ofEpochDay(255))), + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.ofEpochDay(255), null)), generateBytes(-1, 0)); - Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.ofEpochDay(0x8000))), + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.ofEpochDay(0x8000), null)), generateBytes(0, 0x80)); - Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.ofEpochDay(0))), + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.ofEpochDay(0), null)), generateBytes(0, 0)); - Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.ofEpochDay(1))), + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.ofEpochDay(1), null)), generateBytes(1, 0)); - Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.ofEpochDay(Short.MAX_VALUE))), + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.ofEpochDay(Short.MAX_VALUE), null)), generateBytes(0xFF, 0x7F)); - Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.ofEpochDay(0xFFFF))), + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.ofEpochDay(0xFFFF), null)), generateBytes(0xFF, 0xFF)); - Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.of(2021, 8, 7))), + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate(o, LocalDate.of(2021, 8, 7), null)), generateBytes(0x9E, 0x49)); } @Test(groups = { "unit" }) public void testReadDate32() throws IOException { - Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xFF, 0xFF, 0xFF, 0xFF)), + Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xFF, 0xFF, 0xFF, 0xFF), null), + LocalDate.ofEpochDay(-1)); + Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0, 0, 0, 0), null), LocalDate.ofEpochDay(0)); + Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(1, 0, 0, 0), null), LocalDate.ofEpochDay(1)); + Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0x17, 0x61, 0, 0), null), + LocalDate.of(2038, 1, 19)); + + Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xCC, 0xBF, 0xFF, 0xFF), null), + LocalDate.of(1925, 1, 1)); + Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xCB, 0xBF, 1, 0), null), + LocalDate.of(2283, 11, 11)); + } + + @Test(dataProvider = "timeZoneProvider", groups = { "unit" }) + public void testReadDate32WithTimeZone(String timeZoneId) throws IOException { + TimeZone tz = TimeZone.getTimeZone(timeZoneId); + Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xFF, 0xFF, 0xFF, 0xFF), tz), LocalDate.ofEpochDay(-1)); - Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0, 0, 0, 0)), LocalDate.ofEpochDay(0)); - Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(1, 0, 0, 0)), LocalDate.ofEpochDay(1)); - Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0x17, 0x61, 0, 0)), LocalDate.of(2038, 1, 19)); + Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0, 0, 0, 0), tz), LocalDate.ofEpochDay(0)); + Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(1, 0, 0, 0), tz), LocalDate.ofEpochDay(1)); + Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0x17, 0x61, 0, 0), tz), + LocalDate.of(2038, 1, 19)); - Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xCC, 0xBF, 0xFF, 0xFF)), + Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xCC, 0xBF, 0xFF, 0xFF), tz), LocalDate.of(1925, 1, 1)); - Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xCB, 0xBF, 1, 0)), LocalDate.of(2283, 11, 11)); + Assert.assertEquals(BinaryStreamUtils.readDate32(generateInput(0xCB, 0xBF, 1, 0), tz), + LocalDate.of(2283, 11, 11)); } @Test(groups = { "unit" }) public void testWriteDate32() throws IOException { - Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.ofEpochDay(-1))), + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.ofEpochDay(-1), null)), generateBytes(0xFF, 0xFF, 0xFF, 0xFF)); - Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.ofEpochDay(0))), + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.ofEpochDay(0), null)), generateBytes(0, 0, 0, 0)); - Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.ofEpochDay(1))), + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.ofEpochDay(1), null)), generateBytes(1, 0, 0, 0)); - Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2038, 1, 19))), + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2038, 1, 19), null)), generateBytes(0x17, 0x61, 0, 0)); - Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(1925, 1, 1))), + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(1925, 1, 1), null)), generateBytes(0xCC, 0xBF, 0xFF, 0xFF)); - Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2283, 11, 11))), + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2283, 11, 11), null)), generateBytes(0xCB, 0xBF, 1, 0)); Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes( - o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(1925, 1, 1).minus(1L, ChronoUnit.DAYS)))); + o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(1925, 1, 1).minus(1L, ChronoUnit.DAYS), null))); Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes( - o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2283, 11, 11).plus(1L, ChronoUnit.DAYS)))); + o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2283, 11, 11).plus(1L, ChronoUnit.DAYS), null))); + } + + @Test(dataProvider = "timeZoneProvider", groups = { "unit" }) + public void testWriteDate32WithTimeZone(String timeZoneId) throws IOException { + TimeZone tz = TimeZone.getTimeZone(timeZoneId); + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.ofEpochDay(-1), tz)), + generateBytes(0xFF, 0xFF, 0xFF, 0xFF)); + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.ofEpochDay(0), tz)), + generateBytes(0, 0, 0, 0)); + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.ofEpochDay(1), tz)), + generateBytes(1, 0, 0, 0)); + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2038, 1, 19), tz)), + generateBytes(0x17, 0x61, 0, 0)); + + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(1925, 1, 1), tz)), + generateBytes(0xCC, 0xBF, 0xFF, 0xFF)); + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2283, 11, 11), tz)), + generateBytes(0xCB, 0xBF, 1, 0)); + + Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes( + o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(1925, 1, 1).minus(1L, ChronoUnit.DAYS), tz))); + Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes( + o -> BinaryStreamUtils.writeDate32(o, LocalDate.of(2283, 11, 11).plus(1L, ChronoUnit.DAYS), tz))); } @Test(groups = { "unit" }) public void testReadDateTime32() throws IOException { - Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(-1, 0, 0, 0)), + Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(-1, 0, 0, 0), null), LocalDateTime.ofEpochSecond(255, 0, ZoneOffset.UTC)); - Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(0, 0, 0, 0x80)), LocalDateTime + Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(0, 0, 0, 0x80), null), LocalDateTime .ofEpochSecond(new BigInteger(1, generateBytes(0x80, 0, 0, 0)).longValue(), 0, ZoneOffset.UTC)); - Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(0, 0, 0, 0)), + Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(0, 0, 0, 0), null), LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC)); - Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(1, 0, 0, 0)), + Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(1, 0, 0, 0), null), LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC)); - Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(0xFF, 0xFF, 0xFF, 0x7F)), + Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(0xFF, 0xFF, 0xFF, 0x7F), null), LocalDateTime.ofEpochSecond(Integer.MAX_VALUE, 0, ZoneOffset.UTC)); - Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(0x2D, 0x9A, 0x0E, 0x61)), + Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(0x2D, 0x9A, 0x0E, 0x61), null), LocalDateTime.of(2021, 8, 7, 14, 35, 25)); } + @Test(dataProvider = "timeZoneProvider", groups = { "unit" }) + public void testReadDateTime32WithTimeZone(String timeZoneId) throws IOException { + TimeZone tz = TimeZone.getTimeZone(timeZoneId); + Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(-1, 0, 0, 0), tz), + LocalDateTime.ofInstant(Instant.ofEpochSecond(255L), tz.toZoneId())); + Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(0, 0, 0, 0x80), tz), LocalDateTime.ofInstant( + Instant.ofEpochSecond(new BigInteger(1, generateBytes(0x80, 0, 0, 0)).longValue()), tz.toZoneId())); + Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(0, 0, 0, 0), tz), + LocalDateTime.ofInstant(Instant.ofEpochSecond(0L), tz.toZoneId())); + Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(1, 0, 0, 0), tz), + LocalDateTime.ofInstant(Instant.ofEpochSecond(1L), tz.toZoneId())); + Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(0xFF, 0xFF, 0xFF, 0x7F), tz), + LocalDateTime.ofInstant(Instant.ofEpochSecond(Integer.MAX_VALUE), tz.toZoneId())); + + Assert.assertEquals(BinaryStreamUtils.readDateTime32(generateInput(0x2D, 0x9A, 0x0E, 0x61), tz), + LocalDateTime.ofInstant( + Instant.ofEpochSecond(new BigInteger(1, generateBytes(0x61, 0x0E, 0x9A, 0x2D)).longValue()), + tz.toZoneId())); + } + @Test(groups = { "unit" }) public void testWriteDateTime32() throws IOException { - Assert.assertEquals( - getWrittenBytes( - o -> BinaryStreamUtils.writeDateTime32(o, LocalDateTime.ofEpochSecond(255, 0, ZoneOffset.UTC))), + Assert.assertEquals(getWrittenBytes( + o -> BinaryStreamUtils.writeDateTime32(o, LocalDateTime.ofEpochSecond(255, 0, ZoneOffset.UTC), null)), generateBytes(-1, 0, 0, 0)); Assert.assertEquals( getWrittenBytes(o -> BinaryStreamUtils.writeDateTime32(o, LocalDateTime.ofEpochSecond( - new BigInteger(1, generateBytes(0x80, 0, 0, 0)).longValue(), 0, ZoneOffset.UTC))), + new BigInteger(1, generateBytes(0x80, 0, 0, 0)).longValue(), 0, ZoneOffset.UTC), null)), generateBytes(0, 0, 0, 0x80)); - Assert.assertEquals( - getWrittenBytes( - o -> BinaryStreamUtils.writeDateTime32(o, LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC))), + Assert.assertEquals(getWrittenBytes( + o -> BinaryStreamUtils.writeDateTime32(o, LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC), null)), generateBytes(0, 0, 0, 0)); - Assert.assertEquals( - getWrittenBytes( - o -> BinaryStreamUtils.writeDateTime32(o, LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC))), + Assert.assertEquals(getWrittenBytes( + o -> BinaryStreamUtils.writeDateTime32(o, LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC), null)), generateBytes(1, 0, 0, 0)); Assert.assertEquals( getWrittenBytes(o -> BinaryStreamUtils.writeDateTime32(o, - LocalDateTime.ofEpochSecond(Integer.MAX_VALUE, 0, ZoneOffset.UTC))), + LocalDateTime.ofEpochSecond(Integer.MAX_VALUE, 0, ZoneOffset.UTC), null)), generateBytes(0xFF, 0xFF, 0xFF, 0x7F)); Assert.assertEquals( - getWrittenBytes(o -> BinaryStreamUtils.writeDateTime32(o, LocalDateTime.of(2021, 8, 7, 14, 35, 25))), + getWrittenBytes( + o -> BinaryStreamUtils.writeDateTime32(o, LocalDateTime.of(2021, 8, 7, 14, 35, 25), null)), generateBytes(0x2D, 0x9A, 0x0E, 0x61)); Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes( - o -> BinaryStreamUtils.writeDateTime32(o, LocalDateTime.ofEpochSecond(-1L, 0, ZoneOffset.UTC)))); + o -> BinaryStreamUtils.writeDateTime32(o, LocalDateTime.ofEpochSecond(-1L, 0, ZoneOffset.UTC), null))); Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes(o -> BinaryStreamUtils.writeDateTime32(o, - LocalDateTime.ofEpochSecond(BinaryStreamUtils.DATETIME_MAX + 1, 0, ZoneOffset.UTC)))); + LocalDateTime.ofEpochSecond(BinaryStreamUtils.DATETIME_MAX + 1, 0, ZoneOffset.UTC), null))); + } + + @Test(dataProvider = "timeZoneProvider", groups = { "unit" }) + public void testWriteDateTime32WithTimeZone(String timeZoneId) throws IOException { + TimeZone tz = TimeZone.getTimeZone(timeZoneId); + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime32(o, + LocalDateTime.ofInstant(Instant.ofEpochSecond(255L), tz.toZoneId()), tz)), + generateBytes(-1, 0, 0, 0)); + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDateTime32(o, + LocalDateTime.ofInstant( + Instant.ofEpochSecond(new BigInteger(1, generateBytes(0x80, 0, 0, 0)).longValue()), + tz.toZoneId()), + tz)), generateBytes(0, 0, 0, 0x80)); + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime32(o, + LocalDateTime.ofInstant(Instant.ofEpochSecond(0L), tz.toZoneId()), tz)), + generateBytes(0, 0, 0, 0)); + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime32(o, + LocalDateTime.ofInstant(Instant.ofEpochSecond(1L), tz.toZoneId()), tz)), + generateBytes(1, 0, 0, 0)); + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime32(o, + LocalDateTime.ofInstant(Instant.ofEpochSecond(Integer.MAX_VALUE), tz.toZoneId()), tz)), + generateBytes(0xFF, 0xFF, 0xFF, 0x7F)); + + Assert.assertEquals(getWrittenBytes(o -> BinaryStreamUtils.writeDateTime32(o, + LocalDateTime.ofInstant( + Instant.ofEpochSecond(new BigInteger(1, generateBytes(0x61, 0x0E, 0x9A, 0x2D)).longValue()), + tz.toZoneId()), + tz)), generateBytes(0x2D, 0x9A, 0x0E, 0x61)); + + Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes(o -> BinaryStreamUtils + .writeDateTime32(o, LocalDateTime.ofInstant(Instant.ofEpochSecond(-1L), tz.toZoneId()), tz))); + Assert.assertThrows(IllegalArgumentException.class, + () -> getWrittenBytes(o -> BinaryStreamUtils.writeDateTime32(o, LocalDateTime + .ofInstant(Instant.ofEpochSecond(BinaryStreamUtils.DATETIME_MAX + 1), tz.toZoneId()), tz))); } @Test(groups = { "unit" }) public void testReadDateTime64() throws IOException { - Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(-1, 0, 0, 0, 0, 0, 0, 0), 0), + Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(-1, 0, 0, 0, 0, 0, 0, 0), 0, null), LocalDateTime.ofEpochSecond(255, 0, ZoneOffset.UTC)); - Assert.assertEquals( - BinaryStreamUtils.readDateTime64(generateInput(0xF6, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF), 1), + Assert.assertEquals(BinaryStreamUtils + .readDateTime64(generateInput(0xF6, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF), 1, null), LocalDateTime.ofEpochSecond(-1L, 0, ZoneOffset.UTC)); - Assert.assertEquals( - BinaryStreamUtils.readDateTime64(generateInput(0xF5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF), 1), + Assert.assertEquals(BinaryStreamUtils + .readDateTime64(generateInput(0xF5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF), 1, null), LocalDateTime.ofEpochSecond(-2L, 900000000, ZoneOffset.UTC)); // Actually query "select toDateTime64(-1.000000001::Decimal64(9), 9)" returns: // 1969-12-31 23:59:59.000000001 // see https://github.com/ClickHouse/ClickHouse/issues/29386 - Assert.assertEquals( - BinaryStreamUtils.readDateTime64(generateInput(0xFF, 0x35, 0x65, 0xC4, 0xFF, 0xFF, 0xFF, 0xFF), 9), + Assert.assertEquals(BinaryStreamUtils + .readDateTime64(generateInput(0xFF, 0x35, 0x65, 0xC4, 0xFF, 0xFF, 0xFF, 0xFF), 9, null), LocalDateTime.ofEpochSecond(-2L, 999999999, ZoneOffset.UTC)); - Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(0, 0, 0, 0, 0, 0, 0, 0), 0), + Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(0, 0, 0, 0, 0, 0, 0, 0), 0, null), LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC)); - Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(1, 0, 0, 0, 0, 0, 0, 0), 0), + Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(1, 0, 0, 0, 0, 0, 0, 0), 0, null), LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC)); - Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(0x0A, 0, 0, 0, 0, 0, 0, 0), 1), + Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(0x0A, 0, 0, 0, 0, 0, 0, 0), 1, null), LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC)); - Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(1, 0, 0, 0, 0, 0, 0, 0), 9), + Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(1, 0, 0, 0, 0, 0, 0, 0), 9, null), LocalDateTime.ofEpochSecond(0L, 1, ZoneOffset.UTC)); } + @Test(dataProvider = "timeZoneProvider", groups = { "unit" }) + public void testReadDateTime64WithTimeZone(String timeZoneId) throws IOException { + TimeZone tz = TimeZone.getTimeZone(timeZoneId); + Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(-1, 0, 0, 0, 0, 0, 0, 0), 0, tz), + LocalDateTime.ofInstant(Instant.ofEpochSecond(255L), tz.toZoneId())); + Assert.assertEquals( + BinaryStreamUtils.readDateTime64(generateInput(0xF6, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF), 1, tz), + LocalDateTime.ofEpochSecond(-1L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) + .atZoneSameInstant(tz.toZoneId()).toLocalDateTime()); + Assert.assertEquals( + BinaryStreamUtils.readDateTime64(generateInput(0xF5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF), 1, tz), + LocalDateTime.ofEpochSecond(-2L, 900000000, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) + .atZoneSameInstant(tz.toZoneId()).toLocalDateTime()); + // Actually query "select toDateTime64(-1.000000001::Decimal64(9), 9)" returns: + // 1969-12-31 23:59:59.000000001 + // see https://github.com/ClickHouse/ClickHouse/issues/29386 + Assert.assertEquals( + BinaryStreamUtils.readDateTime64(generateInput(0xFF, 0x35, 0x65, 0xC4, 0xFF, 0xFF, 0xFF, 0xFF), 9, tz), + LocalDateTime.ofEpochSecond(-2L, 999999999, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) + .atZoneSameInstant(tz.toZoneId()).toLocalDateTime()); + Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(0, 0, 0, 0, 0, 0, 0, 0), 0, tz), + LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) + .atZoneSameInstant(tz.toZoneId()).toLocalDateTime()); + Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(1, 0, 0, 0, 0, 0, 0, 0), 0, tz), + LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) + .atZoneSameInstant(tz.toZoneId()).toLocalDateTime()); + + Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(0x0A, 0, 0, 0, 0, 0, 0, 0), 1, tz), + LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) + .atZoneSameInstant(tz.toZoneId()).toLocalDateTime()); + Assert.assertEquals(BinaryStreamUtils.readDateTime64(generateInput(1, 0, 0, 0, 0, 0, 0, 0), 9, tz), + LocalDateTime.ofEpochSecond(0L, 1, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) + .atZoneSameInstant(tz.toZoneId()).toLocalDateTime()); + } + @Test(groups = { "unit" }) public void testWriteDateTime64() throws IOException { - Assert.assertEquals(getWrittenBytes( - o -> BinaryStreamUtils.writeDateTime64(o, LocalDateTime.ofEpochSecond(255L, 0, ZoneOffset.UTC), 0)), + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, + LocalDateTime.ofEpochSecond(255L, 0, ZoneOffset.UTC), 0, null)), generateBytes(-1, 0, 0, 0, 0, 0, 0, 0)); - Assert.assertEquals(getWrittenBytes( - o -> BinaryStreamUtils.writeDateTime64(o, LocalDateTime.ofEpochSecond(-1L, 0, ZoneOffset.UTC), 1)), + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, + LocalDateTime.ofEpochSecond(-1L, 0, ZoneOffset.UTC), 1, null)), generateBytes(0xF6, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF)); Assert.assertEquals( getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, - LocalDateTime.ofEpochSecond(-2L, 900000000, ZoneOffset.UTC), 1)), + LocalDateTime.ofEpochSecond(-2L, 900000000, ZoneOffset.UTC), 1, null)), generateBytes(0xF5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF)); Assert.assertEquals( getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, - LocalDateTime.ofEpochSecond(-2L, 999999999, ZoneOffset.UTC), 9)), + LocalDateTime.ofEpochSecond(-2L, 999999999, ZoneOffset.UTC), 9, null)), generateBytes(0xFF, 0x35, 0x65, 0xC4, 0xFF, 0xFF, 0xFF, 0xFF)); Assert.assertEquals(getWrittenBytes( - o -> BinaryStreamUtils.writeDateTime64(o, LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC), 0)), + o -> BinaryStreamUtils.writeDateTime64(o, LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC), 0, null)), generateBytes(0, 0, 0, 0, 0, 0, 0, 0)); Assert.assertEquals(getWrittenBytes( - o -> BinaryStreamUtils.writeDateTime64(o, LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC), 0)), + o -> BinaryStreamUtils.writeDateTime64(o, LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC), 0, null)), generateBytes(1, 0, 0, 0, 0, 0, 0, 0)); Assert.assertEquals(getWrittenBytes( - o -> BinaryStreamUtils.writeDateTime64(o, LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC), 1)), + o -> BinaryStreamUtils.writeDateTime64(o, LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC), 1, null)), generateBytes(0x0A, 0, 0, 0, 0, 0, 0, 0)); Assert.assertEquals(getWrittenBytes( - o -> BinaryStreamUtils.writeDateTime64(o, LocalDateTime.ofEpochSecond(0L, 1, ZoneOffset.UTC), 9)), + o -> BinaryStreamUtils.writeDateTime64(o, LocalDateTime.ofEpochSecond(0L, 1, ZoneOffset.UTC), 9, null)), generateBytes(1, 0, 0, 0, 0, 0, 0, 0)); - Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes( - o -> BinaryStreamUtils.writeDateTime64(o, LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC), -1))); - Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes( - o -> BinaryStreamUtils.writeDateTime64(o, LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC), 10))); + Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes(o -> BinaryStreamUtils + .writeDateTime64(o, LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC), -1, null))); + Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes(o -> BinaryStreamUtils + .writeDateTime64(o, LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC), 10, null))); Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, - LocalDateTime.of(LocalDate.of(1925, 1, 1).minus(1L, ChronoUnit.DAYS), LocalTime.MAX)))); + LocalDateTime.of(LocalDate.of(1925, 1, 1).minus(1L, ChronoUnit.DAYS), LocalTime.MAX), null))); Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, - LocalDateTime.of(LocalDate.of(2283, 11, 11).plus(1L, ChronoUnit.DAYS), LocalTime.MIN)))); + LocalDateTime.of(LocalDate.of(2283, 11, 11).plus(1L, ChronoUnit.DAYS), LocalTime.MIN), null))); + } + + @Test(dataProvider = "timeZoneProvider", groups = { "unit" }) + public void testWriteDateTime64WithTimeZone(String timeZoneId) throws IOException { + TimeZone tz = TimeZone.getTimeZone(timeZoneId); + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, + LocalDateTime.ofInstant(Instant.ofEpochSecond(255L), tz.toZoneId()), 0, tz)), + generateBytes(-1, 0, 0, 0, 0, 0, 0, 0)); + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, + LocalDateTime.ofInstant(Instant.ofEpochSecond(-1L), tz.toZoneId()), 1, tz)), + generateBytes(0xF6, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF)); + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, + LocalDateTime.ofInstant(Instant.ofEpochSecond(-2L, 900000000), tz.toZoneId()), 1, tz)), + generateBytes(0xF5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF)); + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, + LocalDateTime.ofInstant(Instant.ofEpochSecond(-2L, 999999999), tz.toZoneId()), 9, tz)), + generateBytes(0xFF, 0x35, 0x65, 0xC4, 0xFF, 0xFF, 0xFF, 0xFF)); + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, + LocalDateTime.ofInstant(Instant.ofEpochSecond(0L, 0), tz.toZoneId()), 0, tz)), + generateBytes(0, 0, 0, 0, 0, 0, 0, 0)); + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, + LocalDateTime.ofInstant(Instant.ofEpochSecond(1L, 0), tz.toZoneId()), 0, tz)), + generateBytes(1, 0, 0, 0, 0, 0, 0, 0)); + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, + LocalDateTime.ofInstant(Instant.ofEpochSecond(1L, 0), tz.toZoneId()), 1, tz)), + generateBytes(0x0A, 0, 0, 0, 0, 0, 0, 0)); + Assert.assertEquals( + getWrittenBytes(o -> BinaryStreamUtils.writeDateTime64(o, + LocalDateTime.ofInstant(Instant.ofEpochSecond(0L, 1), tz.toZoneId()), 9, tz)), + generateBytes(1, 0, 0, 0, 0, 0, 0, 0)); + + Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes(o -> BinaryStreamUtils + .writeDateTime64(o, LocalDateTime.ofInstant(Instant.ofEpochSecond(0L, 0), tz.toZoneId()), -1, tz))); + Assert.assertThrows(IllegalArgumentException.class, () -> getWrittenBytes(o -> BinaryStreamUtils + .writeDateTime64(o, LocalDateTime.ofInstant(Instant.ofEpochSecond(0L, 0), tz.toZoneId()), 10, tz))); + + Assert.assertThrows(IllegalArgumentException.class, + () -> getWrittenBytes( + o -> BinaryStreamUtils.writeDateTime64(o, + LocalDateTime.of(LocalDate.of(1925, 1, 1).minus(1L, ChronoUnit.DAYS), LocalTime.MAX) + .atOffset(ZoneOffset.UTC).atZoneSameInstant(tz.toZoneId()).toLocalDateTime(), + tz))); + Assert.assertThrows(IllegalArgumentException.class, + () -> getWrittenBytes( + o -> BinaryStreamUtils.writeDateTime64(o, + LocalDateTime.of(LocalDate.of(2283, 11, 11).plus(1L, ChronoUnit.DAYS), LocalTime.MIN) + .atOffset(ZoneOffset.UTC).atZoneSameInstant(tz.toZoneId()).toLocalDateTime(), + tz))); } @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseBitmapTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseBitmapTest.java new file mode 100644 index 000000000..441fe2715 --- /dev/null +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseBitmapTest.java @@ -0,0 +1,57 @@ +package com.clickhouse.client.data; + +import com.clickhouse.client.ClickHouseDataType; + +import org.roaringbitmap.RoaringBitmap; +import org.roaringbitmap.buffer.ImmutableRoaringBitmap; +import org.roaringbitmap.buffer.MutableRoaringBitmap; +import org.roaringbitmap.longlong.Roaring64Bitmap; +import org.roaringbitmap.longlong.Roaring64NavigableMap; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseBitmapTest { + @Test(groups = "unit") + public void testEmptyBitmap32() { + byte[] expectedBytes = new byte[] { 0, 0 }; + ClickHouseDataType[] types = new ClickHouseDataType[] { ClickHouseDataType.Int8, ClickHouseDataType.UInt8, + ClickHouseDataType.Int16, ClickHouseDataType.UInt16, ClickHouseDataType.Int32, + ClickHouseDataType.UInt32 }; + for (ClickHouseDataType t : types) { + Assert.assertTrue(ClickHouseBitmap.empty(t).isEmpty(), "Bitmap should be empty"); + Assert.assertEquals(ClickHouseBitmap.empty(t).toBytes(), expectedBytes); + } + + Object[] bitmaps = new Object[] { RoaringBitmap.bitmapOf(), MutableRoaringBitmap.bitmapOf(), + ImmutableRoaringBitmap.bitmapOf() }; + + for (Object bm : bitmaps) { + for (ClickHouseDataType t : types) { + ClickHouseBitmap v = ClickHouseBitmap.wrap(bm, t); + Assert.assertTrue(v.isEmpty(), "Bitmap should be empty"); + Assert.assertEquals(v.toBytes(), expectedBytes); + } + } + } + + @Test(groups = "unit") + public void testEmptyBitmap64() { + byte[] expectedBytes = new byte[] { 0, 0 }; + + Assert.assertTrue(ClickHouseBitmap.empty(ClickHouseDataType.Int64).isEmpty(), "Bitmap should be empty"); + Assert.assertEquals(ClickHouseBitmap.empty(ClickHouseDataType.Int64).toBytes(), expectedBytes); + Assert.assertTrue(ClickHouseBitmap.empty(ClickHouseDataType.UInt64).isEmpty(), "Bitmap should be empty"); + Assert.assertEquals(ClickHouseBitmap.empty(ClickHouseDataType.UInt64).toBytes(), expectedBytes); + + ClickHouseDataType[] types = new ClickHouseDataType[] { ClickHouseDataType.Int64, ClickHouseDataType.UInt64 }; + Object[] bitmaps = new Object[] { Roaring64Bitmap.bitmapOf(), Roaring64NavigableMap.bitmapOf() }; + + for (Object bm : bitmaps) { + for (ClickHouseDataType t : types) { + ClickHouseBitmap v = ClickHouseBitmap.wrap(bm, t); + Assert.assertTrue(v.isEmpty(), "Bitmap should be empty"); + Assert.assertEquals(v.toBytes(), expectedBytes); + } + } + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseByteValueTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseByteValueTest.java index 67d93e611..fa4db3792 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseByteValueTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseByteValueTest.java @@ -180,7 +180,7 @@ public void testValue() throws Exception { Inet6Address.getAllByName("0:0:0:0:0:0:0:ff")[0], // Inet6Address "-1", // String "-1", // SQL Expression - java.time.DateTimeException.class, // Time + LocalTime.of(23, 59, 59), // Time UUID.fromString("00000000-0000-0000-ffff-ffffffffffff"), // UUID Integer.class, // Key class Byte.class, // Value class diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseDoubleValueTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseDoubleValueTest.java index cf6090fe8..8609728e4 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseDoubleValueTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseDoubleValueTest.java @@ -45,7 +45,7 @@ public void testValue() throws Exception { NumberFormatException.class, // Inet6Address "NaN", // String "NaN", // SQL Expression - LocalTime.ofSecondOfDay(0), // Time + NumberFormatException.class, // Time NumberFormatException.class, // UUID Object.class, // Key class Double.class, // Value class @@ -77,7 +77,7 @@ public void testValue() throws Exception { NumberFormatException.class, // Inet6Address "Infinity", // String "Inf", // SQL Expression - DateTimeException.class, // Time + NumberFormatException.class, // Time NumberFormatException.class, // UUID Object.class, // Key class Double.class, // Value class @@ -109,7 +109,7 @@ public void testValue() throws Exception { NumberFormatException.class, // Inet6Address "-Infinity", // String "-Inf", // SQL Expression - DateTimeException.class, // Time + NumberFormatException.class, // Time NumberFormatException.class, // UUID Object.class, // Key class Double.class, // Value class @@ -240,7 +240,7 @@ public void testValue() throws Exception { Inet6Address.getAllByName("0:0:0:0:0:0:0:ff")[0], // Inet6Address "-1.0", // String "-1.0", // SQL Expression - java.time.DateTimeException.class, // Time + LocalTime.of(23, 59, 59), // Time UUID.fromString("00000000-0000-0000-ffff-ffffffffffff"), // UUID Object.class, // Key class Double.class, // Value class diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseFloatValueTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseFloatValueTest.java index 428da210e..fcd9ad98a 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseFloatValueTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseFloatValueTest.java @@ -45,7 +45,7 @@ public void testValue() throws Exception { NumberFormatException.class, // Inet6Address "NaN", // String "NaN", // SQL Expression - LocalTime.ofSecondOfDay(0), // Time + NumberFormatException.class, // Time NumberFormatException.class, // UUID Object.class, // Key class Float.class, // Value class @@ -77,7 +77,7 @@ public void testValue() throws Exception { NumberFormatException.class, // Inet6Address "Infinity", // String "Inf", // SQL Expression - DateTimeException.class, // Time + NumberFormatException.class, // Time NumberFormatException.class, // UUID Object.class, // Key class Float.class, // Value class @@ -109,7 +109,7 @@ public void testValue() throws Exception { NumberFormatException.class, // Inet6Address "-Infinity", // String "-Inf", // SQL Expression - DateTimeException.class, // Time + NumberFormatException.class, // Time NumberFormatException.class, // UUID Object.class, // Key class Float.class, // Value class @@ -240,7 +240,7 @@ public void testValue() throws Exception { Inet6Address.getAllByName("0:0:0:0:0:0:0:ff")[0], // Inet6Address "-1.0", // String "-1.0", // SQL Expression - java.time.DateTimeException.class, // Time + LocalTime.of(23, 59, 59), // Time UUID.fromString("00000000-0000-0000-ffff-ffffffffffff"), // UUID Object.class, // Key class Float.class, // Value class diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseIntegerValueTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseIntegerValueTest.java index daffb3e01..d2589dead 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseIntegerValueTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseIntegerValueTest.java @@ -141,7 +141,7 @@ public void testValue() throws Exception { Inet6Address.getAllByName("0:0:0:0:0:0:0:ff")[0], // Inet6Address "-1", // String "-1", // SQL Expression - java.time.DateTimeException.class, // Time + LocalTime.of(23, 59, 59), // Time UUID.fromString("00000000-0000-0000-ffff-ffffffffffff"), // UUID Object.class, // Key class Integer.class, // Value class diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseLongValueTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseLongValueTest.java index cfbc191ab..490666037 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseLongValueTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseLongValueTest.java @@ -14,7 +14,6 @@ import org.testng.annotations.Test; import com.clickhouse.client.BaseClickHouseValueTest; import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseValue; import com.clickhouse.client.ClickHouseValues; public class ClickHouseLongValueTest extends BaseClickHouseValueTest { @@ -147,7 +146,7 @@ public void testSignedValue() throws Exception { Inet6Address.getAllByName("0:0:0:0:0:0:0:ff")[0], // Inet6Address "-1", // String "-1", // SQL Expression - java.time.DateTimeException.class, // Time + LocalTime.of(23, 59, 59), // Time UUID.fromString("00000000-0000-0000-ffff-ffffffffffff"), // UUID Object.class, // Key class Long.class, // Value class @@ -292,7 +291,7 @@ public void testUnsignedValue() throws Exception { Inet6Address.getAllByName("0:0:0:0:ffff:ffff:ffff:ffff")[0], // Inet6Address "18446744073709551615", // String "18446744073709551615", // SQL Expression - java.time.DateTimeException.class, // Time + LocalTime.of(23, 59, 59), // Time UUID.fromString("00000000-0000-0000-ffff-ffffffffffff"), // UUID Object.class, // Key class Long.class, // Value class diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseOffsetDateTimeValueTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseOffsetDateTimeValueTest.java new file mode 100644 index 000000000..b848dc6d4 --- /dev/null +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseOffsetDateTimeValueTest.java @@ -0,0 +1,230 @@ +package com.clickhouse.client.data; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.TimeZone; +import java.util.UUID; + +import org.testng.Assert; +import org.testng.annotations.Test; +import com.clickhouse.client.BaseClickHouseValueTest; +import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.client.ClickHouseValues; + +public class ClickHouseOffsetDateTimeValueTest extends BaseClickHouseValueTest { + @Test(groups = { "unit" }) + public void testUpdate() { + Assert.assertEquals(ClickHouseOffsetDateTimeValue.ofNull(0, null).update(-1L).getValue(), + LocalDateTime.ofEpochSecond(-1L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC)); + Assert.assertEquals(ClickHouseOffsetDateTimeValue.ofNull(0, null).update(-1.1F).getValue(), + LocalDateTime.ofEpochSecond(-1L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC)); + Assert.assertEquals(ClickHouseOffsetDateTimeValue.ofNull(3, null).update(-1L).getValue(), + LocalDateTime.ofEpochSecond(-1L, 999000000, ZoneOffset.UTC).atOffset(ZoneOffset.UTC)); + Assert.assertEquals(ClickHouseOffsetDateTimeValue.ofNull(3, null).update(-1.1F).getValue(), + LocalDateTime.ofEpochSecond(-2L, 900000000, ZoneOffset.UTC).atOffset(ZoneOffset.UTC)); + + Assert.assertEquals( + ClickHouseOffsetDateTimeValue.ofNull(9, null).update(new BigDecimal(BigInteger.ONE, 9)).getValue(), + LocalDateTime.ofEpochSecond(0L, 1, ZoneOffset.UTC).atOffset(ZoneOffset.UTC)); + Assert.assertEquals( + ClickHouseOffsetDateTimeValue.ofNull(9, null).update(new BigDecimal(BigInteger.valueOf(-1L), 9)) + .getValue(), + LocalDateTime.ofEpochSecond(-1L, 999999999, ZoneOffset.UTC).atOffset(ZoneOffset.UTC)); + } + + @Test(groups = { "unit" }) + public void testValueWithoutScale() throws Exception { + // null value + checkNull(ClickHouseOffsetDateTimeValue.ofNull(0, null)); + checkNull(ClickHouseOffsetDateTimeValue.of(LocalDateTime.now(), 0, null).resetToNullOrEmpty()); + + // non-null + checkValue(ClickHouseOffsetDateTimeValue.of(LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC), 0, null), false, // isInfinity + false, // isNan + false, // isNull + false, // boolean + (byte) 0, // byte + (short) 0, // short + 0, // int + 0L, // long + 0F, // float + 0D, // double + BigDecimal.valueOf(0L), // BigDecimal + new BigDecimal(BigInteger.ZERO, 3), // BigDecimal + BigInteger.ZERO, // BigInteger + ClickHouseDataType.values()[0].name(), // Enum + LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC), // Object + LocalDate.ofEpochDay(0L), // Date + LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC), // DateTime + LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC), // DateTime(9) + Inet4Address.getAllByName("0.0.0.0")[0], // Inet4Address + Inet6Address.getAllByName("0:0:0:0:0:0:0:0")[0], // Inet6Address + "1970-01-01 00:00:00", // String + "'1970-01-01 00:00:00'", // SQL Expression + LocalTime.ofSecondOfDay(0L), // Time + UUID.fromString("00000000-0000-0000-0000-000000000000"), // UUID + Object.class, // Key class + OffsetDateTime.class, // Value class + new Object[] { LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) }, // Array + new OffsetDateTime[] { LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) }, // typed + // Array + buildMap(new Object[] { 1 }, + new Object[] { LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) }), // Map + buildMap(new Object[] { 1 }, + new OffsetDateTime[] { + LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) }), // typed + // Map + Arrays.asList(LocalDateTime.ofEpochSecond(0L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC)) // Tuple + ); + checkValue(ClickHouseOffsetDateTimeValue.of(LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC), 0, null), false, // isInfinity + false, // isNan + false, // isNull + true, // boolean + (byte) 1, // byte + (short) 1, // short + 1, // int + 1L, // long + 1F, // float + 1D, // double + BigDecimal.valueOf(1L), // BigDecimal + new BigDecimal(BigInteger.ONE, 3), // BigDecimal + BigInteger.ONE, // BigInteger + ClickHouseDataType.values()[1].name(), // Enum + LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC), // Object + LocalDate.ofEpochDay(0L), // Date + LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC), // DateTime + LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC), // DateTime(9) + Inet4Address.getAllByName("0.0.0.1")[0], // Inet4Address + Inet6Address.getAllByName("0:0:0:0:0:0:0:1")[0], // Inet6Address + "1970-01-01 00:00:01", // String + "'1970-01-01 00:00:01'", // SQL Expression + LocalTime.ofSecondOfDay(1L), // Time + UUID.fromString("00000000-0000-0000-0000-000000000001"), // UUID + Object.class, // Key class + OffsetDateTime.class, // Value class + new Object[] { LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) }, // Array + new OffsetDateTime[] { LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) }, // typed + // Array + buildMap(new Object[] { 1 }, + new Object[] { LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) }), // Map + buildMap(new Object[] { 1 }, + new OffsetDateTime[] { + LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) }), // typed + // Map + Arrays.asList(LocalDateTime.ofEpochSecond(1L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC)) // Tuple + ); + checkValue(ClickHouseOffsetDateTimeValue.of(LocalDateTime.ofEpochSecond(2L, 0, ZoneOffset.UTC), 0, null), false, // isInfinity + false, // isNan + false, // isNull + IllegalArgumentException.class, // boolean + (byte) 2, // byte + (short) 2, // short + 2, // int + 2L, // long + 2F, // float + 2D, // double + BigDecimal.valueOf(2L), // BigDecimal + new BigDecimal(BigInteger.valueOf(2L), 3), // BigDecimal + BigInteger.valueOf(2L), // BigInteger + ClickHouseDataType.values()[2].name(), // Enum + LocalDateTime.ofEpochSecond(2L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC), // Object + LocalDate.ofEpochDay(0L), // Date + LocalDateTime.ofEpochSecond(2L, 0, ZoneOffset.UTC), // DateTime + LocalDateTime.ofEpochSecond(2L, 0, ZoneOffset.UTC), // DateTime(9) + Inet4Address.getAllByName("0.0.0.2")[0], // Inet4Address + Inet6Address.getAllByName("0:0:0:0:0:0:0:2")[0], // Inet6Address + "1970-01-01 00:00:02", // String + "'1970-01-01 00:00:02'", // SQL Expression + LocalTime.ofSecondOfDay(2L), // Time + UUID.fromString("00000000-0000-0000-0000-000000000002"), // UUID + Object.class, // Key class + OffsetDateTime.class, // Value class + new Object[] { LocalDateTime.ofEpochSecond(2L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) }, // Array + new OffsetDateTime[] { LocalDateTime.ofEpochSecond(2L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) }, // typed + // Array + buildMap(new Object[] { 1 }, + new Object[] { LocalDateTime.ofEpochSecond(2L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) }), // Map + buildMap(new Object[] { 1 }, + new OffsetDateTime[] { + LocalDateTime.ofEpochSecond(2L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC) }), // typed + // Map + Arrays.asList(LocalDateTime.ofEpochSecond(2L, 0, ZoneOffset.UTC).atOffset(ZoneOffset.UTC)) // Tuple + ); + } + + @Test(groups = { "unit" }) + public void testValueWithScale() throws Exception { + // null value + checkNull(ClickHouseOffsetDateTimeValue.ofNull(3, null)); + checkNull(ClickHouseOffsetDateTimeValue.of(LocalDateTime.now(), 9, null).resetToNullOrEmpty()); + + // non-null + OffsetDateTime dateTime = LocalDateTime.ofEpochSecond(0L, 123456789, ZoneOffset.UTC).atOffset(ZoneOffset.UTC); + checkValue(ClickHouseOffsetDateTimeValue.of(dateTime.toLocalDateTime(), 3, null), false, // isInfinity + false, // isNan + false, // isNull + false, // boolean + (byte) 0, // byte + (short) 0, // short + 0, // int + 0L, // long + 0.123456789F, // float + 0.123456789D, // double + BigDecimal.valueOf(0L), // BigDecimal + BigDecimal.valueOf(0.123D), // BigDecimal + BigInteger.ZERO, // BigInteger + ClickHouseDataType.values()[0].name(), // Enum + dateTime, // Object + LocalDate.ofEpochDay(0L), // Date + dateTime.toLocalDateTime(), // DateTime + dateTime.toLocalDateTime(), // DateTime(9) + Inet4Address.getAllByName("0.0.0.0")[0], // Inet4Address + Inet6Address.getAllByName("0:0:0:0:0:0:0:0")[0], // Inet6Address + "1970-01-01 00:00:00.123456789", // String + "'1970-01-01 00:00:00.123456789'", // SQL Expression + LocalTime.ofNanoOfDay(123456789L), // Time + UUID.fromString("00000000-0000-0000-0000-000000000000"), // UUID + Object.class, // Key class + OffsetDateTime.class, // Value class + new Object[] { dateTime }, // Array + new OffsetDateTime[] { dateTime }, // typed Array + buildMap(new Object[] { 1 }, new Object[] { dateTime }), // Map + buildMap(new Object[] { 1 }, new OffsetDateTime[] { dateTime }), // typed + // Map + Arrays.asList(dateTime) // Tuple + ); + Assert.assertEquals(ClickHouseOffsetDateTimeValue.of(dateTime.toLocalDateTime(), 3, null).asBigDecimal(4), + BigDecimal.valueOf(0.1235D)); + } + + @Test(groups = { "unit" }) + public void testValueWithTimeZone() throws Exception { + LocalDateTime dateTime = LocalDateTime.of(2020, 2, 11, 0, 23, 33); + TimeZone tz = null; + ClickHouseOffsetDateTimeValue v = ClickHouseOffsetDateTimeValue.of(dateTime, 0, tz); + Assert.assertEquals(v.asDateTime(0), dateTime); + Assert.assertEquals(v.asOffsetDateTime(0), OffsetDateTime.of(dateTime, ZoneOffset.UTC)); + Assert.assertEquals(ClickHouseValues.convertToSqlExpression(v.asOffsetDateTime()), v.toSqlExpression()); + v = ClickHouseOffsetDateTimeValue.of(dateTime, 0, tz = TimeZone.getTimeZone("UTC")); + Assert.assertEquals(v.asDateTime(0), dateTime); + Assert.assertEquals(v.asOffsetDateTime(0), ZonedDateTime.of(dateTime, tz.toZoneId()).toOffsetDateTime()); + Assert.assertEquals(ClickHouseValues.convertToSqlExpression(v.asOffsetDateTime()), v.toSqlExpression()); + v = ClickHouseOffsetDateTimeValue.of(dateTime, 0, tz = TimeZone.getTimeZone("Asia/Shanghai")); + Assert.assertEquals(v.asDateTime(0), dateTime); + Assert.assertEquals(v.asOffsetDateTime(0), ZonedDateTime.of(dateTime, tz.toZoneId()).toOffsetDateTime()); + Assert.assertEquals(ClickHouseValues.convertToSqlExpression(v.asOffsetDateTime()), v.toSqlExpression()); + v = ClickHouseOffsetDateTimeValue.of(dateTime, 0, tz = TimeZone.getTimeZone("America/Los_Angeles")); + Assert.assertEquals(v.asDateTime(0), dateTime); + Assert.assertEquals(v.asOffsetDateTime(0), ZonedDateTime.of(dateTime, tz.toZoneId()).toOffsetDateTime()); + Assert.assertEquals(ClickHouseValues.convertToSqlExpression(v.asOffsetDateTime()), v.toSqlExpression()); + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessorTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessorTest.java index 0eb7b27f8..c06ab3e9f 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessorTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessorTest.java @@ -1,9 +1,12 @@ package com.clickhouse.client.data; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigInteger; import java.net.InetAddress; +import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.testng.Assert; @@ -11,6 +14,8 @@ import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.client.data.array.ClickHouseByteArrayValue; +import com.clickhouse.client.data.array.ClickHouseShortArrayValue; public class ClickHouseRowBinaryProcessorTest { private ClickHouseRowBinaryProcessor newProcessor(int... bytes) throws IOException { @@ -20,37 +25,36 @@ private ClickHouseRowBinaryProcessor newProcessor(int... bytes) throws IOExcepti @Test(groups = { "unit" }) public void testDeserializeArray() throws IOException { - - ClickHouseValue value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize( - ClickHouseColumn.of("a", "Array(UInt8)"), null, BinaryStreamUtilsTest.generateInput(2, 1, 2)); - Assert.assertTrue(value instanceof ClickHouseArrayValue); - Short[] shortArray = value.asObject(Short[].class); + ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseValue value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize(null, config, + ClickHouseColumn.of("a", "Array(UInt8)"), BinaryStreamUtilsTest.generateInput(2, 1, 2)); + Assert.assertTrue(value instanceof ClickHouseShortArrayValue); + Assert.assertEquals(value.asObject(), new short[] { 1, 2 }); + Object[] shortArray = value.asArray(); Assert.assertEquals(shortArray.length, 2); Assert.assertEquals(shortArray[0], Short.valueOf("1")); Assert.assertEquals(shortArray[1], Short.valueOf("2")); - value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize( - ClickHouseColumn.of("a", "Array(Nullable(Int8))"), null, - BinaryStreamUtilsTest.generateInput(2, 0, 1, 0, 2)); - Assert.assertTrue(value instanceof ClickHouseArrayValue); - Byte[] byteArray = value.asObject(Byte[].class); + value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize(null, config, + ClickHouseColumn.of("a", "Array(Nullable(Int8))"), BinaryStreamUtilsTest.generateInput(2, 0, 1, 0, 2)); + Assert.assertTrue(value instanceof ClickHouseByteArrayValue); + Assert.assertEquals(value.asObject(), new byte[] { 1, 2 }); + Object[] byteArray = value.asArray(); Assert.assertEquals(byteArray.length, 2); Assert.assertEquals(byteArray[0], Byte.valueOf("1")); Assert.assertEquals(byteArray[1], Byte.valueOf("2")); - value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize( - ClickHouseColumn.of("a", "Array(Array(UInt8))"), null, BinaryStreamUtilsTest.generateInput(1, 2, 1, 2)); + value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize(null, config, + ClickHouseColumn.of("a", "Array(Array(UInt8))"), BinaryStreamUtilsTest.generateInput(1, 2, 1, 2)); Assert.assertTrue(value instanceof ClickHouseArrayValue); + Assert.assertEquals(value.asObject(), new short[][] { new short[] { 1, 2 } }); Object[] array = (Object[]) value.asObject(); Assert.assertEquals(array.length, 1); - shortArray = (Short[]) array[0]; - Assert.assertEquals(shortArray.length, 2); - Assert.assertEquals(shortArray[0], Short.valueOf("1")); - Assert.assertEquals(shortArray[1], Short.valueOf("2")); + Assert.assertEquals(array[0], new short[] { 1, 2 }); // SELECT arrayZip(['a', 'b', 'c'], [3, 2, 1]) - value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize( - ClickHouseColumn.of("a", "Array(Tuple(String, UInt8))"), null, + value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize(null, config, + ClickHouseColumn.of("a", "Array(Tuple(String, UInt8))"), BinaryStreamUtilsTest.generateInput(3, 1, 0x61, 3, 1, 0x62, 2, 1, 0x63, 1)); Assert.assertTrue(value instanceof ClickHouseArrayValue); array = (Object[]) value.asObject(); @@ -66,8 +70,8 @@ public void testDeserializeArray() throws IOException { Assert.assertEquals(((List) array[2]).get(1), Short.valueOf("1")); // insert into x values([{ 'a' : (null, 3), 'b' : (1, 2), 'c' : (2, 1)}]) - value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize( - ClickHouseColumn.of("a", "Array(Map(String, Tuple(Nullable(UInt8), UInt16)))"), null, + value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize(null, config, + ClickHouseColumn.of("a", "Array(Map(String, Tuple(Nullable(UInt8), UInt16)))"), BinaryStreamUtilsTest.generateInput(1, 3, 1, 0x61, 1, 3, 0, 1, 0x62, 0, 1, 2, 0, 1, 0x63, 0, 2, 1, 0)); Assert.assertTrue(value instanceof ClickHouseArrayValue); array = (Object[]) value.asObject(); @@ -88,20 +92,65 @@ public void testDeserializeArray() throws IOException { Assert.assertEquals(l.get(1), 1); } + @Test(groups = { "unit" }) + public void testSerializeArray() throws IOException { + ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseValue value = ClickHouseShortArrayValue.of(new short[] { 1, 2 }); + ByteArrayOutputStream bas = new ByteArrayOutputStream(); + ClickHouseRowBinaryProcessor.getMappedFunctions().serialize(value, config, + ClickHouseColumn.of("a", "Array(UInt8)"), bas); + Assert.assertEquals(bas.toByteArray(), BinaryStreamUtilsTest.generateBytes(2, 1, 2)); + + value = ClickHouseByteArrayValue.of(new byte[] { 1, 2 }); + bas = new ByteArrayOutputStream(); + ClickHouseRowBinaryProcessor.getMappedFunctions().serialize(value, config, + ClickHouseColumn.of("a", "Array(Nullable(Int8))"), bas); + Assert.assertEquals(bas.toByteArray(), BinaryStreamUtilsTest.generateBytes(2, 0, 1, 0, 2)); + + value = ClickHouseArrayValue.of(new short[][] { new short[] { 1, 2 } }); + bas = new ByteArrayOutputStream(); + ClickHouseRowBinaryProcessor.getMappedFunctions().serialize(value, config, + ClickHouseColumn.of("a", "Array(Array(UInt8))"), bas); + Assert.assertEquals(bas.toByteArray(), BinaryStreamUtilsTest.generateBytes(1, 2, 1, 2)); + + // SELECT arrayZip(['a', 'b', 'c'], [3, 2, 1]) + value = ClickHouseArrayValue.of(new Object[] { Arrays.asList("a", (short) 3), Arrays.asList("b", (short) 2), + Arrays.asList("c", (short) 1) }); + bas = new ByteArrayOutputStream(); + ClickHouseRowBinaryProcessor.getMappedFunctions().serialize(value, config, + ClickHouseColumn.of("a", "Array(Tuple(String, UInt8))"), bas); + Assert.assertEquals(bas.toByteArray(), + BinaryStreamUtilsTest.generateBytes(3, 1, 0x61, 3, 1, 0x62, 2, 1, 0x63, 1)); + + // insert into x values([{ 'a' : (null, 3), 'b' : (1, 2), 'c' : (2, 1)}]) + value = ClickHouseArrayValue.of(new Object[] { new LinkedHashMap>() { + { + put("a", Arrays.asList((Short) null, 3)); + put("b", Arrays.asList(Short.valueOf("1"), 2)); + put("c", Arrays.asList(Short.valueOf("2"), 1)); + } + } }); + bas = new ByteArrayOutputStream(); + ClickHouseRowBinaryProcessor.getMappedFunctions().serialize(value, config, + ClickHouseColumn.of("a", "Array(Map(String, Tuple(Nullable(UInt8), UInt16)))"), bas); + Assert.assertEquals(bas.toByteArray(), + BinaryStreamUtilsTest.generateBytes(1, 3, 1, 0x61, 1, 3, 0, 1, 0x62, 0, 1, 2, 0, 1, 0x63, 0, 2, 1, 0)); + } + @Test(groups = { "unit" }) public void testDeserializeMap() throws IOException { - ClickHouseValue value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize( - ClickHouseColumn.of("m", "Map(UInt8, UInt8)"), null, - BinaryStreamUtilsTest.generateInput(2, 2, 2, 1, 1)); + ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseValue value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize(null, config, + ClickHouseColumn.of("m", "Map(UInt8, UInt8)"), BinaryStreamUtilsTest.generateInput(2, 2, 2, 1, 1)); Assert.assertTrue(value instanceof ClickHouseMapValue); Map map = (Map) value.asObject(); Assert.assertEquals(map.size(), 2); Assert.assertEquals(map.get((short) 2), (short) 2); Assert.assertEquals(map.get((short) 1), (short) 1); - value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize( - ClickHouseColumn.of("m", "Map(String, UInt32)"), null, - BinaryStreamUtilsTest.generateInput(2, 1, 0x32, 2, 0, 0, 0, 1, 0x31, 1, 0, 0, 0, 0)); + value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize(null, config, + ClickHouseColumn.of("m", "Map(String, UInt32)"), + BinaryStreamUtilsTest.generateInput(2, 1, 0x32, 2, 0, 0, 0, 1, 0x31, 1, 0, 0, 0)); Assert.assertTrue(value instanceof ClickHouseMapValue); map = (Map) value.asObject(); Assert.assertEquals(map.size(), 2); @@ -109,10 +158,38 @@ public void testDeserializeMap() throws IOException { Assert.assertEquals(map.get("1"), 1L); } + @Test(groups = { "unit" }) + public void testSerializeMap() throws IOException { + ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseValue value = ClickHouseMapValue.of(new LinkedHashMap() { + { + put((short) 2, (short) 2); + put((short) 1, (short) 1); + } + }, Short.class, Short.class); + ByteArrayOutputStream bas = new ByteArrayOutputStream(); + ClickHouseRowBinaryProcessor.getMappedFunctions().serialize(value, config, + ClickHouseColumn.of("m", "Map(UInt8, UInt8)"), bas); + Assert.assertEquals(bas.toByteArray(), BinaryStreamUtilsTest.generateBytes(2, 2, 2, 1, 1)); + + value = ClickHouseMapValue.of(new LinkedHashMap() { + { + put("2", 2L); + put("1", 1L); + } + }, String.class, Long.class); + bas = new ByteArrayOutputStream(); + ClickHouseRowBinaryProcessor.getMappedFunctions().serialize(value, config, + ClickHouseColumn.of("m", "Map(String, UInt32)"), bas); + Assert.assertEquals(bas.toByteArray(), + BinaryStreamUtilsTest.generateBytes(2, 1, 0x32, 2, 0, 0, 0, 1, 0x31, 1, 0, 0, 0)); + } + @Test(groups = { "unit" }) public void testDeserializeNested() throws IOException { - ClickHouseValue value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize( - ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Int16)"), null, + ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseValue value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize(null, config, + ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Int16)"), BinaryStreamUtilsTest.generateInput(1, 1, 1, 0, 1, 0x32, 1, 3, 0)); Assert.assertTrue(value instanceof ClickHouseNestedValue); @@ -128,21 +205,33 @@ public void testDeserializeNested() throws IOException { Assert.assertEquals(values[2], new Short[] { Short.valueOf("3") }); } + @Test(groups = { "unit" }) + public void testSerializeNested() throws IOException { + ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseValue value = ClickHouseNestedValue.of( + ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Int16)").getNestedColumns(), + new Object[][] { new Short[] { Short.valueOf("1") }, new String[] { "2" }, + new Short[] { Short.valueOf("3") } }); + ByteArrayOutputStream bas = new ByteArrayOutputStream(); + ClickHouseRowBinaryProcessor.getMappedFunctions().serialize(value, config, + ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Int16)"), bas); + Assert.assertEquals(bas.toByteArray(), BinaryStreamUtilsTest.generateBytes(1, 1, 1, 0, 1, 0x32, 1, 3, 0)); + } + @Test(groups = { "unit" }) public void testDeserializeTuple() throws IOException { - ClickHouseValue value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize( - ClickHouseColumn.of("t", "Tuple(UInt8, String)"), null, - BinaryStreamUtilsTest.generateInput(1, 1, 0x61)); + ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseValue value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize(null, config, + ClickHouseColumn.of("t", "Tuple(UInt8, String)"), BinaryStreamUtilsTest.generateInput(1, 1, 0x61)); Assert.assertTrue(value instanceof ClickHouseTupleValue); List values = (List) value.asObject(); Assert.assertEquals(values.size(), 2); Assert.assertEquals(values.get(0), (short) 1); Assert.assertEquals(values.get(1), "a"); - value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize( - ClickHouseColumn.of("t", "Tuple(UInt32, Int128, Nullable(IPv4)))"), value, - BinaryStreamUtilsTest.generateInput(1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0x05, 0xa8, 0xc0)); + value = ClickHouseRowBinaryProcessor.getMappedFunctions().deserialize(value, config, + ClickHouseColumn.of("t", "Tuple(UInt32, Int128, Nullable(IPv4)))"), BinaryStreamUtilsTest.generateInput( + 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0x05, 0xa8, 0xc0)); Assert.assertTrue(value instanceof ClickHouseTupleValue); values = (List) value.asObject(); Assert.assertEquals(values.size(), 3); @@ -150,4 +239,21 @@ public void testDeserializeTuple() throws IOException { Assert.assertEquals(values.get(1), BigInteger.valueOf(2)); Assert.assertEquals(values.get(2), InetAddress.getByName("192.168.5.1")); } + + @Test(groups = { "unit" }) + public void testSerializeTuple() throws IOException { + ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseValue value = ClickHouseTupleValue.of((byte) 1, "a"); + ByteArrayOutputStream bas = new ByteArrayOutputStream(); + ClickHouseRowBinaryProcessor.getMappedFunctions().serialize(value, config, + ClickHouseColumn.of("t", "Tuple(UInt8, String)"), bas); + Assert.assertEquals(bas.toByteArray(), BinaryStreamUtilsTest.generateBytes(1, 1, 0x61)); + + value = ClickHouseTupleValue.of(1L, BigInteger.valueOf(2), InetAddress.getByName("192.168.5.1")); + bas = new ByteArrayOutputStream(); + ClickHouseRowBinaryProcessor.getMappedFunctions().serialize(value, config, + ClickHouseColumn.of("t", "Tuple(UInt32, Int128, Nullable(IPv4)))"), bas); + Assert.assertEquals(bas.toByteArray(), BinaryStreamUtilsTest.generateBytes(1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0x05, 0xa8, 0xc0)); + } } diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseShortValueTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseShortValueTest.java index 453e9cb2d..b955bcfcf 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseShortValueTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseShortValueTest.java @@ -141,7 +141,7 @@ public void testValue() throws Exception { Inet6Address.getAllByName("0:0:0:0:0:0:0:ff")[0], // Inet6Address "-1", // String "-1", // SQL Expression - java.time.DateTimeException.class, // Time + LocalTime.of(23, 59, 59), // Time UUID.fromString("00000000-0000-0000-ffff-ffffffffffff"), // UUID Object.class, // Key class Short.class, // Value class diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseSimpleRecordTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseSimpleRecordTest.java new file mode 100644 index 000000000..653fbf9dd --- /dev/null +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseSimpleRecordTest.java @@ -0,0 +1,79 @@ +package com.clickhouse.client.data; + +import java.util.Arrays; +import java.util.Collections; + +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.ClickHouseValue; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseSimpleRecordTest { + @Test(groups = { "unit" }) + public void testNullInput() { + Assert.assertThrows(IllegalArgumentException.class, () -> ClickHouseSimpleRecord.of(null, null)); + Assert.assertThrows(IllegalArgumentException.class, + () -> ClickHouseSimpleRecord.of(null, new ClickHouseValue[0])); + Assert.assertThrows(IllegalArgumentException.class, + () -> ClickHouseSimpleRecord.of(Collections.emptyList(), null)); + + ClickHouseSimpleRecord record = new ClickHouseSimpleRecord(null, null); + Assert.assertNull(record.getColumns()); + Assert.assertNull(record.getValues()); + } + + @Test(groups = { "unit" }) + public void testMismatchedColumnsAndValues() { + Assert.assertThrows(IllegalArgumentException.class, () -> ClickHouseSimpleRecord + .of(Arrays.asList(ClickHouseColumn.of("a", "String")), new ClickHouseValue[0])); + + ClickHouseSimpleRecord record = new ClickHouseSimpleRecord(Arrays.asList(ClickHouseColumn.of("a", "String")), + new ClickHouseValue[0]); + Assert.assertEquals(record.getColumns(), Arrays.asList(ClickHouseColumn.of("a", "String"))); + Assert.assertEquals(record.getValues(), new ClickHouseValue[0]); + } + + @Test(groups = { "unit" }) + public void testGetValueByIndex() { + ClickHouseSimpleRecord record = new ClickHouseSimpleRecord(ClickHouseColumn.parse("a String, b UInt32"), + new ClickHouseValue[] { ClickHouseStringValue.of("123"), ClickHouseLongValue.of(1L, true) }); + Assert.assertEquals(record.getColumns(), ClickHouseColumn.parse("a String, b UInt32")); + Assert.assertEquals(record.getValues(), + new ClickHouseValue[] { ClickHouseStringValue.of("123"), ClickHouseLongValue.of(1L, true) }); + + Assert.assertEquals(record.getValue(0), ClickHouseStringValue.of("123")); + Assert.assertEquals(record.getValue(1), ClickHouseLongValue.of(1L, true)); + Assert.assertThrows(ArrayIndexOutOfBoundsException.class, () -> record.getValue(-1)); + Assert.assertThrows(ArrayIndexOutOfBoundsException.class, () -> record.getValue(2)); + + int index = 0; + for (ClickHouseValue v : record) { + if (index == 0) { + Assert.assertEquals(v, ClickHouseStringValue.of("123")); + } else { + Assert.assertEquals(v, ClickHouseLongValue.of(1L, true)); + } + index++; + } + } + + @Test(groups = { "unit" }) + public void testGetValueByName() { + ClickHouseSimpleRecord record = new ClickHouseSimpleRecord( + ClickHouseColumn.parse("`a One` String, `x木哈哈x` UInt32, test Nullable(String)"), + new ClickHouseValue[] { ClickHouseStringValue.of("123"), ClickHouseLongValue.of(1L, true), + ClickHouseStringValue.ofNull() }); + Assert.assertEquals(record.getColumns(), + ClickHouseColumn.parse("`a One` String, `x木哈哈x` UInt32, test Nullable(String)")); + Assert.assertEquals(record.getValues(), new ClickHouseValue[] { ClickHouseStringValue.of("123"), + ClickHouseLongValue.of(1L, true), ClickHouseStringValue.ofNull() }); + + Assert.assertEquals(record.getValue("A one"), ClickHouseStringValue.of("123")); + Assert.assertEquals(record.getValue("x木哈哈x"), ClickHouseLongValue.of(1L, true)); + Assert.assertEquals(record.getValue("TEST"), ClickHouseStringValue.ofNull()); + + Assert.assertThrows(IllegalArgumentException.class, () -> record.getValue(null)); + Assert.assertThrows(IllegalArgumentException.class, () -> record.getValue("non-exist")); + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseSimpleResponseTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseSimpleResponseTest.java new file mode 100644 index 000000000..3c97b40cf --- /dev/null +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseSimpleResponseTest.java @@ -0,0 +1,114 @@ +package com.clickhouse.client.data; + +import java.util.Collections; +import java.util.List; +import java.util.NoSuchElementException; + +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.ClickHouseRecord; +import com.clickhouse.client.ClickHouseResponse; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseSimpleResponseTest { + @Test(groups = { "unit" }) + public void testNullOrEmptyInput() { + ClickHouseResponse nullResp = ClickHouseSimpleResponse.of((List) null, null); + Assert.assertEquals(nullResp.getColumns(), Collections.emptyList()); + Assert.assertTrue(((List) nullResp.records()).isEmpty()); + Assert.assertThrows(NoSuchElementException.class, () -> nullResp.firstRecord()); + + ClickHouseResponse emptyResp1 = ClickHouseSimpleResponse.of(ClickHouseColumn.parse("a String"), null); + Assert.assertEquals(emptyResp1.getColumns(), ClickHouseColumn.parse("a String")); + Assert.assertTrue(((List) emptyResp1.records()).isEmpty()); + Assert.assertThrows(NoSuchElementException.class, () -> emptyResp1.firstRecord()); + + ClickHouseResponse emptyResp2 = ClickHouseSimpleResponse.of(ClickHouseColumn.parse("a String"), + new Object[0][]); + Assert.assertEquals(emptyResp2.getColumns(), ClickHouseColumn.parse("a String")); + Assert.assertTrue(((List) emptyResp2.records()).isEmpty()); + Assert.assertThrows(NoSuchElementException.class, () -> emptyResp2.firstRecord()); + } + + @Test(groups = { "unit" }) + public void testMismatchedColumnsAndRecords() { + ClickHouseResponse resp = ClickHouseSimpleResponse + .of(ClickHouseColumn.parse("a Nullable(String), b UInt8, c Array(UInt32)"), + new Object[][] { new Object[0], null, new Object[] { 's' }, + new Object[] { null, null, null, null }, + new Object[] { "123", 1, new int[] { 3, 2, 1 } } }); + int i = 0; + for (ClickHouseRecord r : resp.records()) { + switch (i) { + case 0: + case 1: + case 3: + Assert.assertNull(r.getValue(0).asObject()); + Assert.assertNull(r.getValue(1).asObject()); + Assert.assertEquals(r.getValue(2).asObject(), new long[0]); + break; + case 2: + Assert.assertEquals(r.getValue(0).asObject(), "s"); + Assert.assertNull(r.getValue(1).asObject()); + Assert.assertEquals(r.getValue(2).asObject(), new long[0]); + break; + case 4: + Assert.assertEquals(r.getValue(0).asObject(), "123"); + Assert.assertEquals(r.getValue(1).asObject(), (short) 1); + Assert.assertEquals(r.getValue(2).asObject(), new long[] { 3L, 2L, 1L }); + break; + default: + Assert.fail("Should not fail"); + break; + } + i++; + } + } + + @Test(groups = { "unit" }) + public void testFirstRecord() { + ClickHouseResponse resp = ClickHouseSimpleResponse.of( + ClickHouseColumn.parse("a Nullable(String), b UInt8, c String"), + new Object[][] { new Object[] { "aaa", 2, "ccc" }, null }); + ClickHouseRecord record = resp.firstRecord(); + Assert.assertEquals(record.getValue("A"), ClickHouseStringValue.of("aaa")); + Assert.assertEquals(record.getValue("B"), ClickHouseShortValue.of(2)); + Assert.assertEquals(record.getValue("C"), ClickHouseStringValue.of("ccc")); + + ClickHouseRecord sameRecord = resp.firstRecord(); + Assert.assertTrue(record == sameRecord); + } + + @Test(groups = { "unit" }) + public void testRecords() { + ClickHouseResponse resp = ClickHouseSimpleResponse.of( + ClickHouseColumn.parse("a Nullable(String), b UInt8, c String"), + new Object[][] { new Object[] { "aaa1", null, "ccc1" }, new Object[] { "aaa2", 2, "ccc2" }, + new Object[] { null, 3L, null } }); + int i = 0; + for (ClickHouseRecord r : resp.records()) { + switch (i) { + case 0: + Assert.assertEquals(r.getValue(0).asObject(), "aaa1"); + Assert.assertNull(r.getValue(1).asObject()); + Assert.assertEquals(r.getValue(2).asObject(), "ccc1"); + break; + case 1: + Assert.assertEquals(r.getValue("a").asObject(), "aaa2"); + Assert.assertEquals(r.getValue("B").asObject(), (short) 2); + Assert.assertEquals(r.getValue("c").asObject(), "ccc2"); + break; + case 2: + Assert.assertNull(r.getValue(0).asObject()); + Assert.assertEquals(r.getValue(1).asObject(), (short) 3); + Assert.assertNull(r.getValue(0).asObject()); + break; + default: + Assert.fail("Should not fail"); + break; + } + i++; + } + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseTimeValueTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseTimeValueTest.java deleted file mode 100644 index d45f7ddb9..000000000 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseTimeValueTest.java +++ /dev/null @@ -1,121 +0,0 @@ -package com.clickhouse.client.data; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.net.Inet4Address; -import java.net.Inet6Address; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.util.Arrays; -import java.util.UUID; -import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseDataType; - -public class ClickHouseTimeValueTest extends BaseClickHouseValueTest { - @Test(groups = { "unit" }) - public void testValue() throws Exception { - // null value - checkNull(ClickHouseTimeValue.ofNull()); - checkNull(ClickHouseTimeValue.of(LocalTime.now()).resetToNullOrEmpty()); - - // non-null - checkValue(ClickHouseTimeValue.of(0), false, // isInfinity - false, // isNan - false, // isNull - false, // boolean - (byte) 0, // byte - (short) 0, // short - 0, // int - 0L, // long - 0F, // float - 0D, // double - BigDecimal.valueOf(0L), // BigDecimal - new BigDecimal(BigInteger.ZERO, 3), // BigDecimal - BigInteger.ZERO, // BigInteger - ClickHouseDataType.values()[0].name(), // Enum - LocalTime.ofSecondOfDay(0L), // Object - LocalDate.ofEpochDay(0L), // Date - LocalDateTime.of(LocalDate.ofEpochDay(0L), LocalTime.ofSecondOfDay(0L)), // DateTime - LocalDateTime.of(LocalDate.ofEpochDay(0L), LocalTime.ofSecondOfDay(0L)), // DateTime(9) - Inet4Address.getAllByName("0.0.0.0")[0], // Inet4Address - Inet6Address.getAllByName("0:0:0:0:0:0:0:0")[0], // Inet6Address - "00:00:00", // String - "'00:00:00'", // SQL Expression - LocalTime.ofSecondOfDay(0L), // Time - UUID.fromString("00000000-0000-0000-0000-000000000000"), // UUID - Object.class, // Key class - LocalTime.class, // Value class - new Object[] { LocalTime.ofSecondOfDay(0L) }, // Array - new LocalTime[] { LocalTime.ofSecondOfDay(0L) }, // typed Array - buildMap(new Object[] { 1 }, new Object[] { LocalTime.ofSecondOfDay(0L) }), // Map - buildMap(new Object[] { 1 }, new LocalTime[] { LocalTime.ofSecondOfDay(0L) }), // typed Map - Arrays.asList(LocalTime.ofSecondOfDay(0L)) // Tuple - ); - checkValue(ClickHouseTimeValue.of(1), false, // isInfinity - false, // isNan - false, // isNull - true, // boolean - (byte) 1, // byte - (short) 1, // short - 1, // int - 1L, // long - 1F, // float - 1D, // double - BigDecimal.valueOf(1L), // BigDecimal - new BigDecimal(BigInteger.ONE, 3), // BigDecimal - BigInteger.ONE, // BigInteger - ClickHouseDataType.values()[1].name(), // Enum - LocalTime.ofSecondOfDay(1L), // Object - LocalDate.ofEpochDay(0L), // Date - LocalDateTime.of(LocalDate.ofEpochDay(0L), LocalTime.ofSecondOfDay(1L)), // DateTime - LocalDateTime.of(LocalDate.ofEpochDay(0L), LocalTime.ofSecondOfDay(1L)), // DateTime(9) - Inet4Address.getAllByName("0.0.0.1")[0], // Inet4Address - Inet6Address.getAllByName("0:0:0:0:0:0:0:1")[0], // Inet6Address - "00:00:01", // String - "'00:00:01'", // SQL Expression - LocalTime.ofSecondOfDay(1L), // Time - UUID.fromString("00000000-0000-0000-0000-000000000001"), // UUID - Object.class, // Key class - LocalTime.class, // Value class - new Object[] { LocalTime.ofSecondOfDay(1L) }, // Array - new LocalTime[] { LocalTime.ofSecondOfDay(1L) }, // typed Array - buildMap(new Object[] { 1 }, new Object[] { LocalTime.ofSecondOfDay(1L) }), // Map - buildMap(new Object[] { 1 }, new LocalTime[] { LocalTime.ofSecondOfDay(1L) }), // typed Map - Arrays.asList(LocalTime.ofSecondOfDay(1L)) // Tuple - ); - checkValue(ClickHouseTimeValue.of(2), false, // isInfinity - false, // isNan - false, // isNull - IllegalArgumentException.class, // boolean - (byte) 2, // byte - (short) 2, // short - 2, // int - 2L, // long - 2F, // float - 2D, // double - BigDecimal.valueOf(2L), // BigDecimal - new BigDecimal(BigInteger.valueOf(2L), 3), // BigDecimal - BigInteger.valueOf(2L), // BigInteger - ClickHouseDataType.values()[2].name(), // Enum - LocalTime.ofSecondOfDay(2L), // Object - LocalDate.ofEpochDay(0L), // Date - LocalDateTime.of(LocalDate.ofEpochDay(0L), LocalTime.ofSecondOfDay(2L)), // DateTime - LocalDateTime.of(LocalDate.ofEpochDay(0L), LocalTime.ofSecondOfDay(2L)), // DateTime(9) - Inet4Address.getAllByName("0.0.0.2")[0], // Inet4Address - Inet6Address.getAllByName("0:0:0:0:0:0:0:2")[0], // Inet6Address - "00:00:02", // String - "'00:00:02'", // SQL Expression - LocalTime.ofSecondOfDay(2L), // Time - UUID.fromString("00000000-0000-0000-0000-000000000002"), // UUID - Object.class, // Key class - LocalTime.class, // Value class - new Object[] { LocalTime.ofSecondOfDay(2L) }, // Array - new LocalTime[] { LocalTime.ofSecondOfDay(2L) }, // typed Array - buildMap(new Object[] { 1 }, new Object[] { LocalTime.ofSecondOfDay(2L) }), // Map - buildMap(new Object[] { 1 }, new LocalTime[] { LocalTime.ofSecondOfDay(2L) }), // typed Map - Arrays.asList(LocalTime.ofSecondOfDay(2L)) // Tuple - ); - } -} diff --git a/clickhouse-grpc-client/pom.xml b/clickhouse-grpc-client/pom.xml index 023278283..d03afc990 100644 --- a/clickhouse-grpc-client/pom.xml +++ b/clickhouse-grpc-client/pom.xml @@ -15,6 +15,10 @@ gRPC client for ClickHouse https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-grpc-client + + ${project.parent.groupId}.client.internal + + ${project.parent.groupId} @@ -39,15 +43,6 @@ io.grpc grpc-protobuf - - - io.grpc - grpc-netty-shaded - provided - - - io.grpc - grpc-okhttp provided @@ -89,6 +84,186 @@ + + org.apache.maven.plugins + maven-shade-plugin + + + shade + package + + shade + + + true + true + true + shaded + + + com.google + ${shade.base}.google + + + com.squareup.okhttp + ${shade.base}.okhttp + + + okio + ${shade.base}.okio + + + io.grpc + ${shade.base}.grpc + + + io.perfmark + ${shade.base}.perfmark + + + io.opencensus + ${shade.base}.opencensus + + + + + + + + + + *:* + + android/** + google/** + javax/** + org/** + META-INF/MANIFEST.MF + META-INF/maven/** + META-INF/native-image/** + META-INF/versions/** + + + + + + + shade-netty + package + + shade + + + true + true + true + netty + + + com.google + ${shade.base}.google + + + io.grpc + ${shade.base}.grpc + + + io.perfmark + ${shade.base}.perfmark + + + io.opencensus + ${shade.base}.opencensus + + + + + + + + + + *:* + + android/** + com/squareup/** + google/** + io/grpc/okhttp/** + javax/** + okio/** + org/** + META-INF/MANIFEST.MF + META-INF/maven/** + META-INF/native-image/** + META-INF/versions/** + + + + + + + shade-okhttp + package + + shade + + + true + true + true + okhttp + + + com.google + ${shade.base}.google + + + com.squareup.okhttp + ${shade.base}.okhttp + + + okio + ${shade.base}.okio + + + io.grpc + ${shade.base}.grpc + + + io.perfmark + ${shade.base}.perfmark + + + io.opencensus + ${shade.base}.opencensus + + + + + + + + + + *:* + + android/** + google/** + io/grpc/netty/** + javax/** + org/** + META-INF/MANIFEST.MF + META-INF/maven/** + META-INF/native/** + META-INF/native-image/** + META-INF/versions/** + + + + + + + org.codehaus.mojo build-helper-maven-plugin diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java index d6391f883..3c816a908 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java @@ -16,7 +16,7 @@ import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.grpc.config.ClickHouseGrpcClientOption; +import com.clickhouse.client.grpc.config.ClickHouseGrpcOption; import com.clickhouse.client.grpc.impl.ClickHouseGrpc; import com.clickhouse.client.logging.Logger; import com.clickhouse.client.logging.LoggerFactory; @@ -62,7 +62,7 @@ public abstract class ClickHouseGrpcChannelFactory { } public static ClickHouseGrpcChannelFactory getFactory(ClickHouseConfig config, ClickHouseNode server) { - return ((boolean) config.getOption(ClickHouseGrpcClientOption.USE_OKHTTP)) + return ((boolean) config.getOption(ClickHouseGrpcOption.USE_OKHTTP)) ? new OkHttpChannelFactoryImpl(config, server) : new NettyChannelFactoryImpl(config, server); } @@ -160,13 +160,13 @@ protected void setupRetry() { protected void setupMisc() { ManagedChannelBuilder builder = getChannelBuilder(); - if ((boolean) config.getOption(ClickHouseGrpcClientOption.USE_FULL_STREAM_DECOMPRESSION)) { + if ((boolean) config.getOption(ClickHouseGrpcOption.USE_FULL_STREAM_DECOMPRESSION)) { builder.enableFullStreamDecompression(); } // TODO add interceptor to customize retry - builder.maxInboundMessageSize((int) config.getOption(ClickHouseGrpcClientOption.MAX_INBOUND_MESSAGE_SIZE)) - .maxInboundMetadataSize((int) config.getOption(ClickHouseGrpcClientOption.MAX_INBOUND_METADATA_SIZE)); + builder.maxInboundMessageSize((int) config.getOption(ClickHouseGrpcOption.MAX_INBOUND_MESSAGE_SIZE)) + .maxInboundMetadataSize((int) config.getOption(ClickHouseGrpcOption.MAX_INBOUND_METADATA_SIZE)); } public ManagedChannel create() { diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcClient.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcClient.java index 184ccbe0c..779451288 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcClient.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcClient.java @@ -8,31 +8,32 @@ import java.util.Map.Entry; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; -import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import com.google.protobuf.ByteString; import io.grpc.Context; import io.grpc.ManagedChannel; import io.grpc.Status; import io.grpc.StatusException; import io.grpc.stub.StreamObserver; + +import com.clickhouse.client.AbstractClient; import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseCredentials; +import com.clickhouse.client.ClickHouseException; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseProtocol; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.config.ClickHouseOption; import com.clickhouse.client.data.ClickHouseExternalTable; -import com.clickhouse.client.exception.ClickHouseException; +import com.clickhouse.client.grpc.config.ClickHouseGrpcOption; import com.clickhouse.client.grpc.impl.ClickHouseGrpc; +import com.clickhouse.client.grpc.impl.Compression; +import com.clickhouse.client.grpc.impl.CompressionAlgorithm; +import com.clickhouse.client.grpc.impl.CompressionLevel; import com.clickhouse.client.grpc.impl.ExternalTable; import com.clickhouse.client.grpc.impl.NameAndType; import com.clickhouse.client.grpc.impl.QueryInfo; @@ -41,26 +42,56 @@ import com.clickhouse.client.logging.Logger; import com.clickhouse.client.logging.LoggerFactory; -public class ClickHouseGrpcClient implements ClickHouseClient { +public class ClickHouseGrpcClient extends AbstractClient { private static final Logger log = LoggerFactory.getLogger(ClickHouseGrpcClient.class); - private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private static final Compression COMPRESSION_DISABLED = Compression.newBuilder() + .setAlgorithm(CompressionAlgorithm.NO_COMPRESSION).setLevel(CompressionLevel.COMPRESSION_NONE).build(); + + protected static Compression getResultCompression(ClickHouseConfig config) { + if (!config.isCompressServerResponse()) { + return COMPRESSION_DISABLED; + } + + Compression.Builder builder = Compression.newBuilder(); + CompressionAlgorithm algorithm = CompressionAlgorithm.DEFLATE; + CompressionLevel level = CompressionLevel.COMPRESSION_MEDIUM; + switch (config.getDecompressAlgorithmForClientRequest()) { + case NONE: + algorithm = CompressionAlgorithm.NO_COMPRESSION; + break; + case DEFLATE: + break; + case GZIP: + algorithm = CompressionAlgorithm.GZIP; + break; + // case STREAM_GZIP: + default: + log.warn("Unsupported algorithm [%s], change to [%s]", config.getDecompressAlgorithmForClientRequest(), + algorithm); + break; + } - // not going to offload executor to ClickHouseGrpcFuture, as we can manage - // thread pool better here - private final AtomicReference executor = new AtomicReference<>(); + int l = config.getDecompressLevelForClientRequest(); + if (l <= 0) { + level = CompressionLevel.COMPRESSION_NONE; + } else if (l < 3) { + level = CompressionLevel.COMPRESSION_LOW; + } else if (l < 7) { + level = CompressionLevel.COMPRESSION_MEDIUM; + } else { + level = CompressionLevel.COMPRESSION_HIGH; + } - // do NOT use below members directly without ReadWriteLock - private final AtomicReference config = new AtomicReference<>(); - private final AtomicReference server = new AtomicReference<>(); - private final AtomicReference channel = new AtomicReference<>(); + return builder.setAlgorithm(algorithm).setLevel(level).build(); + } protected static QueryInfo convert(ClickHouseNode server, ClickHouseRequest request) { ClickHouseConfig config = request.getConfig(); - ClickHouseCredentials credentials = server.getCredentials().orElse(config.getDefaultCredentials()); + ClickHouseCredentials credentials = server.getCredentials(config); Builder builder = QueryInfo.newBuilder(); - builder.setDatabase(server.getDatabase()).setUserName(credentials.getUserName()) + builder.setDatabase(server.getDatabase(config)).setUserName(credentials.getUserName()) .setPassword(credentials.getPassword()).setOutputFormat(request.getFormat().name()); Optional optionalValue = request.getSessionId(); @@ -79,6 +110,8 @@ protected static QueryInfo convert(ClickHouseNode server, ClickHouseRequest r builder.setQueryId(optionalValue.get()); } + builder.setResultCompression(getResultCompression(config)); + // builder.setNextQueryInfo(true); for (Entry s : request.getSettings().entrySet()) { builder.putSettings(s.getKey(), String.valueOf(s.getValue())); @@ -89,7 +122,7 @@ protected static QueryInfo convert(ClickHouseNode server, ClickHouseRequest r try { builder.setInputData(ByteString.readFrom(input.get())); } catch (IOException e) { - throw new IllegalStateException(e); + throw new CompletionException(ClickHouseException.of(e, server)); } } @@ -108,7 +141,7 @@ protected static QueryInfo convert(ClickHouseNode server, ClickHouseRequest r try { builder.addExternalTables(b.setData(ByteString.readFrom(external.getContent())).build()); } catch (IOException e) { - throw new IllegalStateException(e); + throw new CompletionException(ClickHouseException.of(e, server)); } } } @@ -142,156 +175,50 @@ protected static QueryInfo convert(ClickHouseNode server, ClickHouseRequest r return builder.setQuery(sql).build(); } - protected void fill(ClickHouseRequest request, StreamObserver observer) { - try { - observer.onNext(convert(getServer(), request)); - } finally { - observer.onCompleted(); - } - } - - protected ClickHouseNode getServer() { - lock.readLock().lock(); - try { - return server.get(); - } finally { - lock.readLock().unlock(); - } - } - - protected ManagedChannel getChannel(ClickHouseRequest request) { - boolean prepared = true; - ClickHouseNode newNode = ClickHouseChecker.nonNull(request, "request").getServer(); - - lock.readLock().lock(); - ManagedChannel c = channel.get(); - try { - prepared = c != null && newNode.equals(server.get()); - - if (prepared) { - return c; - } - } finally { - lock.readLock().unlock(); - } - - lock.writeLock().lock(); - c = channel.get(); - try { - // first time? - if (c == null) { - server.set(newNode); - channel.set(c = ClickHouseGrpcChannelFactory.getFactory(getConfig(), newNode).create()); - } else if (!newNode.equals(server.get())) { - log.debug("Shutting down channel: %s", c); - c.shutdownNow(); - - server.set(newNode); - channel.set(c = ClickHouseGrpcChannelFactory.getFactory(getConfig(), newNode).create()); - } - - return c; - } finally { - lock.writeLock().unlock(); - } - } - @Override - public boolean accept(ClickHouseProtocol protocol) { - return ClickHouseProtocol.GRPC == protocol || ClickHouseClient.super.accept(protocol); + protected void closeConnection(ManagedChannel connection, boolean force) { + if (!force) { + connection.shutdown(); + } else { + connection.shutdownNow(); + } } @Override - public ClickHouseConfig getConfig() { - lock.readLock().lock(); - try { - return config.get(); - } finally { - lock.readLock().unlock(); + protected ManagedChannel newConnection(ManagedChannel connection, ClickHouseNode server, + ClickHouseRequest request) { + if (connection != null) { + closeConnection(connection, false); } + + return ClickHouseGrpcChannelFactory.getFactory(request.getConfig(), server).create(); } - @Override - public void init(ClickHouseConfig config) { - lock.writeLock().lock(); + protected void fill(ClickHouseRequest request, StreamObserver observer) { try { - this.config.set(config); - ClickHouseClient.super.init(config); - if (this.executor.get() == null) { // only initialize once - int threads = config.getMaxThreadsPerClient(); - this.executor.set(threads <= 0 ? ClickHouseClient.getExecutorService() - : ClickHouseUtils.newThreadPool(ClickHouseGrpcClient.class.getSimpleName(), threads, - config.getMaxQueuedRequests())); - } + observer.onNext(convert(getServer(), request)); } finally { - lock.writeLock().unlock(); + observer.onCompleted(); } } @Override - public void close() { - lock.writeLock().lock(); - - ExecutorService s = executor.get(); - ManagedChannel m = channel.get(); - - try { - server.set(null); - - if (s != null) { - s.shutdown(); - executor.set(null); - } - - if (m != null) { - m.shutdown(); - channel.set(null); - } - - ClickHouseConfig c = config.get(); - if (c != null) { - config.set(null); - } - - // shutdown* won't shutdown commonPool, so awaitTermination will always time out - // on the other hand, for a client-specific thread pool, we'd better shut it - // down for real - if (s != null && c != null && c.getMaxThreadsPerClient() > 0 - && !s.awaitTermination((int) c.getOption(ClickHouseClientOption.CONNECTION_TIMEOUT), - TimeUnit.MILLISECONDS)) { - s.shutdownNow(); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (RuntimeException e) { - log.warn("Exception occurred when closing client", e); - } finally { - try { - if (m != null) { - m.shutdownNow(); - } - - if (s != null) { - s.shutdownNow(); - } - } finally { - lock.writeLock().unlock(); - } - } + public boolean accept(ClickHouseProtocol protocol) { + return ClickHouseProtocol.GRPC == protocol || super.accept(protocol); } protected CompletableFuture executeAsync(ClickHouseRequest sealedRequest, ManagedChannel channel, ClickHouseNode server) { // reuse stub? ClickHouseGrpc.ClickHouseStub stub = ClickHouseGrpc.newStub(channel); - stub.withCompression(sealedRequest.getCompression().encoding()); + stub.withCompression(sealedRequest.getConfig().getDecompressAlgorithmForClientRequest().encoding()); final ClickHouseStreamObserver responseObserver = new ClickHouseStreamObserver(sealedRequest.getConfig(), server); final StreamObserver requestObserver = stub.executeQueryWithStreamIO(responseObserver); if (sealedRequest.hasInputStream()) { - executor.get().execute(() -> fill(sealedRequest, requestObserver)); + getExecutor().execute(() -> fill(sealedRequest, requestObserver)); } else { fill(sealedRequest, requestObserver); } @@ -311,37 +238,57 @@ protected CompletableFuture executeAsync(ClickHouseRequest executeSync(ClickHouseRequest sealedRequest, - ManagedChannel channel, ClickHouseNode server) throws ClickHouseException { + ManagedChannel channel, ClickHouseNode server) { ClickHouseGrpc.ClickHouseBlockingStub stub = ClickHouseGrpc.newBlockingStub(channel); - stub.withCompression(sealedRequest.getCompression().encoding()); - Result result = stub.executeQuery(convert(server, sealedRequest)); + stub.withCompression(sealedRequest.getConfig().getDecompressAlgorithmForClientRequest().encoding()); // TODO not as elegant as ClickHouseImmediateFuture :< - return CompletableFuture.completedFuture( - new ClickHouseGrpcResponse(sealedRequest.getConfig(), server, sealedRequest.getSettings(), result)); + try { + Result result = stub.executeQuery(convert(server, sealedRequest)); + + ClickHouseResponse response = new ClickHouseGrpcResponse(sealedRequest.getConfig(), + sealedRequest.getSettings(), result); + + return result.hasException() + ? failedResponse(ClickHouseException.of(result.getException().getDisplayText(), server)) + : CompletableFuture.completedFuture(response); + } catch (IOException e) { + throw new CompletionException(ClickHouseException.of(e, server)); + } } @Override - public CompletableFuture execute(ClickHouseRequest request) throws ClickHouseException { + public CompletableFuture execute(ClickHouseRequest request) { // sealedRequest is an immutable copy of the original request final ClickHouseRequest sealedRequest = request.seal(); - final ManagedChannel c = getChannel(sealedRequest); + final ManagedChannel c = getConnection(sealedRequest); final ClickHouseNode s = getServer(); return sealedRequest.getConfig().isAsync() ? executeAsync(sealedRequest, c, s) : executeSync(sealedRequest, c, s); } + + @Override + public Class getOptionClass() { + return ClickHouseGrpcOption.class; + } } diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcFuture.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcFuture.java index 560ee8c6d..5bc0346c6 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcFuture.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcFuture.java @@ -1,5 +1,6 @@ package com.clickhouse.client.grpc; +import java.io.IOException; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -13,7 +14,6 @@ import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.exception.ClickHouseException; import com.clickhouse.client.grpc.impl.QueryInfo; @Deprecated @@ -80,8 +80,8 @@ public ClickHouseResponse get(long timeout, TimeUnit unit) } try { - return new ClickHouseGrpcResponse(request.getConfig(), server, request.getSettings(), responseObserver); - } catch (ClickHouseException e) { + return new ClickHouseGrpcResponse(request.getConfig(), request.getSettings(), responseObserver); + } catch (IOException e) { throw new ExecutionException(e); } } diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcResponse.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcResponse.java index 2cba35e73..3c574c6d5 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcResponse.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcResponse.java @@ -1,95 +1,48 @@ package com.clickhouse.client.grpc; +import java.io.IOException; import java.util.Map; + import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseNode; -import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.exception.ClickHouseException; +import com.clickhouse.client.data.ClickHouseStreamResponse; +import com.clickhouse.client.grpc.impl.Progress; import com.clickhouse.client.grpc.impl.Result; +import com.clickhouse.client.grpc.impl.Stats; -public class ClickHouseGrpcResponse extends ClickHouseResponse { +public class ClickHouseGrpcResponse extends ClickHouseStreamResponse { private final ClickHouseStreamObserver observer; private final Result result; - protected ClickHouseGrpcResponse(ClickHouseConfig config, ClickHouseNode server, Map settings, - ClickHouseStreamObserver observer) throws ClickHouseException { - super(config, server, settings, observer.getInputStream(), null, observer.getError()); + protected ClickHouseGrpcResponse(ClickHouseConfig config, Map settings, + ClickHouseStreamObserver observer) throws IOException { + super(config, observer.getInputStream(), settings, null, observer.getSummary()); this.observer = observer; this.result = null; - - throwErrorIfAny(); } - protected ClickHouseGrpcResponse(ClickHouseConfig config, ClickHouseNode server, Map settings, - Result result) throws ClickHouseException { - super(config, server, settings, result.getOutput().newInput(), null, - result.hasException() - ? new ClickHouseException(result.getException().getCode(), - result.getException().getDisplayText(), null) - : null); + protected ClickHouseGrpcResponse(ClickHouseConfig config, Map settings, Result result) + throws IOException { + super(config, result.getOutput().newInput(), settings, null, new ClickHouseResponseSummary(null, null)); this.observer = null; this.result = result; + if (result.hasProgress()) { + Progress p = result.getProgress(); + summary.update(new ClickHouseResponseSummary.Progress(p.getReadRows(), p.getReadBytes(), + p.getTotalRowsToRead(), p.getWrittenRows(), p.getWrittenBytes())); + } - throwErrorIfAny(); + if (result.hasStats()) { + Stats s = result.getStats(); + summary.update(new ClickHouseResponseSummary.Statistics(s.getRows(), s.getBlocks(), s.getAllocatedBytes(), + s.getAppliedLimit(), s.getRowsBeforeLimit())); + } } @Override public ClickHouseResponseSummary getSummary() { - ClickHouseResponseSummary summary = super.getSummary(); - - if (result != null && (result.hasProgress() || result.hasStats())) { - summary = new ClickHouseResponseSummary() { - @Override - public long getAllocatedBytes() { - return result.getStats().getAllocatedBytes(); - } - - @Override - public long getBlocks() { - return result.getStats().getBlocks(); - } - - @Override - public long getReadBytes() { - return result.getProgress().getReadBytes(); - } - - @Override - public long getReadRows() { - return result.getProgress().getReadRows(); - } - - @Override - public long getRows() { - return result.getStats().getRows(); - } - - @Override - public long getRowsBeforeLimit() { - return result.getStats().getRowsBeforeLimit(); - } - - @Override - public long getTotalRowsToRead() { - return result.getProgress().getTotalRowsToRead(); - } - - @Override - public long getWriteBytes() { - return result.getProgress().getWrittenBytes(); - } - - @Override - public long getWriteRows() { - return result.getProgress().getWrittenRows(); - } - }; - } else if (observer != null) { - summary = observer.getSummary(); - } return summary; } } diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcResponseSummary.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcResponseSummary.java deleted file mode 100644 index 18a39c02d..000000000 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcResponseSummary.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.clickhouse.client.grpc; - -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import com.clickhouse.client.ClickHouseResponseSummary; - -public class ClickHouseGrpcResponseSummary implements ClickHouseResponseSummary { - // grpc - protected final AtomicInteger results; - - // stats - protected final AtomicLong allocatedBytes; - protected final AtomicLong blocks; - protected final AtomicLong rows; - protected final AtomicLong rowsBeforeLimit; - - // progress - protected final AtomicLong readBytes; - protected final AtomicLong readRows; - protected final AtomicLong totalRowsToRead; - protected final AtomicLong writeBytes; - protected final AtomicLong writeRows; - - protected ClickHouseGrpcResponseSummary() { - this.results = new AtomicInteger(0); - - this.readBytes = new AtomicLong(0L); - this.readRows = new AtomicLong(0L); - this.totalRowsToRead = new AtomicLong(0L); - this.writeBytes = new AtomicLong(0L); - this.writeRows = new AtomicLong(0L); - - this.allocatedBytes = new AtomicLong(0L); - this.blocks = new AtomicLong(0L); - this.rows = new AtomicLong(0L); - this.rowsBeforeLimit = new AtomicLong(0L); - } - - public int getResults() { - return results.get(); - } - - @Override - public long getAllocatedBytes() { - return allocatedBytes.get(); - } - - @Override - public long getBlocks() { - return blocks.get(); - } - - @Override - public long getRows() { - return rows.get(); - } - - @Override - public long getRowsBeforeLimit() { - return rowsBeforeLimit.get(); - } - - @Override - public long getReadBytes() { - return readBytes.get(); - } - - @Override - public long getReadRows() { - return readRows.get(); - } - - @Override - public long getTotalRowsToRead() { - return totalRowsToRead.get(); - } - - @Override - public long getWriteBytes() { - return writeBytes.get(); - } - - @Override - public long getWriteRows() { - return writeRows.get(); - } -} diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseStreamObserver.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseStreamObserver.java index da016973b..930940fb4 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseStreamObserver.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseStreamObserver.java @@ -9,10 +9,11 @@ import io.grpc.stub.StreamObserver; import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseDataStreamFactory; +import com.clickhouse.client.ClickHouseException; import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseResponseSummary; import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.data.ClickHousePipedStream; -import com.clickhouse.client.exception.ClickHouseException; import com.clickhouse.client.grpc.impl.Exception; import com.clickhouse.client.grpc.impl.LogEntry; import com.clickhouse.client.grpc.impl.Progress; @@ -31,7 +32,7 @@ public class ClickHouseStreamObserver implements StreamObserver { private final ClickHousePipedStream stream; - private final ClickHouseGrpcResponseSummary summary; + private final ClickHouseResponseSummary summary; private Throwable error; @@ -43,7 +44,7 @@ protected ClickHouseStreamObserver(ClickHouseConfig config, ClickHouseNode serve this.stream = ClickHouseDataStreamFactory.getInstance().createPipedStream(config); - this.summary = new ClickHouseGrpcResponseSummary(); + this.summary = new ClickHouseResponseSummary(null, null); this.error = null; } @@ -61,7 +62,7 @@ protected void setError(Throwable error) { } protected boolean updateStatus(Result result) { - summary.results.incrementAndGet(); + summary.update(); log.info(() -> { for (LogEntry e : result.getLogsList()) { @@ -80,20 +81,15 @@ protected boolean updateStatus(Result result) { boolean proceed = true; if (result.hasStats()) { - Stats stats = result.getStats(); - summary.allocatedBytes.set(stats.getAllocatedBytes()); - summary.blocks.set(stats.getBlocks()); - summary.rows.set(stats.getRows()); - summary.rowsBeforeLimit.set(stats.getRowsBeforeLimit()); + Stats s = result.getStats(); + summary.update(new ClickHouseResponseSummary.Statistics(s.getRows(), s.getBlocks(), s.getAllocatedBytes(), + s.getAppliedLimit(), s.getRowsBeforeLimit())); } if (result.hasProgress()) { - Progress prog = result.getProgress(); - summary.readBytes.set(prog.getReadBytes()); - summary.readRows.set(prog.getReadRows()); - summary.totalRowsToRead.set(prog.getTotalRowsToRead()); - summary.writeBytes.set(prog.getWrittenBytes()); - summary.writeRows.set(prog.getWrittenRows()); + Progress p = result.getProgress(); + summary.update(new ClickHouseResponseSummary.Progress(p.getReadRows(), p.getReadBytes(), + p.getTotalRowsToRead(), p.getWrittenRows(), p.getWrittenBytes())); } if (result.getCancelled()) { @@ -107,7 +103,7 @@ protected boolean updateStatus(Result result) { if (error == null) { error = new ClickHouseException(result.getException().getCode(), result.getException().getDisplayText(), - null); + this.server); } } @@ -122,7 +118,7 @@ public boolean isCancelled() { return isCompleted() && error != null; } - public ClickHouseGrpcResponseSummary getSummary() { + public ClickHouseResponseSummary getSummary() { return summary; } diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/NettyChannelFactoryImpl.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/NettyChannelFactoryImpl.java index 5f2a3e683..d0adb67fa 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/NettyChannelFactoryImpl.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/NettyChannelFactoryImpl.java @@ -16,7 +16,7 @@ import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.config.ClickHouseSslMode; -import com.clickhouse.client.grpc.config.ClickHouseGrpcClientOption; +import com.clickhouse.client.grpc.config.ClickHouseGrpcOption; final class NettyChannelFactoryImpl extends ClickHouseGrpcChannelFactory { private final NettyChannelBuilder builder; @@ -26,7 +26,7 @@ final class NettyChannelFactoryImpl extends ClickHouseGrpcChannelFactory { builder = NettyChannelBuilder.forAddress(server.getHost(), server.getPort()); - int flowControlWindow = (int) config.getOption(ClickHouseGrpcClientOption.FLOW_CONTROL_WINDOW); + int flowControlWindow = (int) config.getOption(ClickHouseGrpcOption.FLOW_CONTROL_WINDOW); if (flowControlWindow > 0) { builder.flowControlWindow(flowControlWindow); // what about initialFlowControlWindow? } diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/OkHttpChannelFactoryImpl.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/OkHttpChannelFactoryImpl.java index ce3386700..19e030473 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/OkHttpChannelFactoryImpl.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/OkHttpChannelFactoryImpl.java @@ -7,7 +7,7 @@ import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseSslContextProvider; -import com.clickhouse.client.grpc.config.ClickHouseGrpcClientOption; +import com.clickhouse.client.grpc.config.ClickHouseGrpcOption; final class OkHttpChannelFactoryImpl extends ClickHouseGrpcChannelFactory { private final OkHttpChannelBuilder builder; @@ -17,7 +17,7 @@ final class OkHttpChannelFactoryImpl extends ClickHouseGrpcChannelFactory { builder = OkHttpChannelBuilder.forAddress(server.getHost(), server.getPort()); - int flowControlWindow = (int) config.getOption(ClickHouseGrpcClientOption.FLOW_CONTROL_WINDOW); + int flowControlWindow = (int) config.getOption(ClickHouseGrpcOption.FLOW_CONTROL_WINDOW); if (flowControlWindow > 0) { builder.flowControlWindow(flowControlWindow); } diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcClientOption.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcOption.java similarity index 78% rename from clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcClientOption.java rename to clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcOption.java index c530646f1..b3f39f214 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcClientOption.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcOption.java @@ -1,21 +1,23 @@ package com.clickhouse.client.grpc.config; +import java.io.Serializable; + import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.config.ClickHouseConfigOption; +import com.clickhouse.client.config.ClickHouseOption; /** * gRPC client options. */ -public enum ClickHouseGrpcClientOption implements ClickHouseConfigOption { +public enum ClickHouseGrpcOption implements ClickHouseOption { /** * Flow control window. */ FLOW_CONTROL_WINDOW("flow_control_window", 0, - "Size of flow control window in byte, 0 or negative number are same as default"), + "Size of flow control window in byte, 0 or negative number are same as default."), /** * Maximum message size. */ - MAX_INBOUND_MESSAGE_SIZE("max_inbound_message_size", 4 * 1024 * 1024, + MAX_INBOUND_MESSAGE_SIZE("max_inbound_message_size", 8 * 1024 * 1024, "The maximum message size allowed to be received."), /** * Maximum size of metadata. @@ -26,7 +28,7 @@ public enum ClickHouseGrpcClientOption implements ClickHouseConfigOption { * Whether to use Okhttp instead of Netty. */ USE_OKHTTP("use_okhttp", false, - "Whether to use lightweight transport based on Okhttp instead of Netty. In many cases Netty is faster than Okhttp."), + "Whether to use lightweight transport based on Okhttp instead of Netty. In most cases Netty is faster than Okhttp."), /** * Whether to use full stream decompression. */ @@ -34,11 +36,11 @@ public enum ClickHouseGrpcClientOption implements ClickHouseConfigOption { "Whether to use full stream decompression for better compression ratio or not."); private final String key; - private final Object defaultValue; + private final Serializable defaultValue; private final Class clazz; private final String description; - ClickHouseGrpcClientOption(String key, T defaultValue, String description) { + ClickHouseGrpcOption(String key, T defaultValue, String description) { this.key = ClickHouseChecker.nonNull(key, "key"); this.defaultValue = ClickHouseChecker.nonNull(defaultValue, "defaultValue"); this.clazz = defaultValue.getClass(); @@ -46,7 +48,7 @@ ClickHouseGrpcClientOption(String key, T defaultValue, String description) { } @Override - public Object getDefaultValue() { + public Serializable getDefaultValue() { return defaultValue; } diff --git a/clickhouse-grpc-client/src/main/java9/module-info.java b/clickhouse-grpc-client/src/main/java9/module-info.java index b8deb302c..90708b66b 100644 --- a/clickhouse-grpc-client/src/main/java9/module-info.java +++ b/clickhouse-grpc-client/src/main/java9/module-info.java @@ -1,15 +1,10 @@ module com.clickhouse.client.grpc { exports com.clickhouse.client.grpc; exports com.clickhouse.client.grpc.config; - // exports com.clickhouse.client.grpc.impl; + exports com.clickhouse.client.grpc.impl; provides com.clickhouse.client.ClickHouseClient with com.clickhouse.client.grpc.ClickHouseGrpcClient; - requires java.base; - - requires static grpc.netty.shaded; - requires static grpc.okhttp; - requires transitive com.clickhouse.client; requires transitive com.google.gson; requires transitive com.google.protobuf; diff --git a/clickhouse-grpc-client/src/main/proto/clickhouse_grpc.proto b/clickhouse-grpc-client/src/main/proto/clickhouse_grpc.proto index 18e054a9e..bd7cf72c6 100644 --- a/clickhouse-grpc-client/src/main/proto/clickhouse_grpc.proto +++ b/clickhouse-grpc-client/src/main/proto/clickhouse_grpc.proto @@ -49,6 +49,25 @@ message ExternalTable { map settings = 5; } +enum CompressionAlgorithm { + NO_COMPRESSION = 0; + DEFLATE = 1; + GZIP = 2; + STREAM_GZIP = 3; +} + +enum CompressionLevel { + COMPRESSION_NONE = 0; + COMPRESSION_LOW = 1; + COMPRESSION_MEDIUM = 2; + COMPRESSION_HIGH = 3; +} + +message Compression { + CompressionAlgorithm algorithm = 1; + CompressionLevel level = 2; +} + // Information about a query which a client sends to a ClickHouse server. // The first QueryInfo can set any of the following fields. Extra QueryInfos only add extra data. // In extra QueryInfos only `input_data`, `external_tables`, `next_query_info` and `cancel` fields can be set. @@ -86,6 +105,10 @@ message QueryInfo { // If true there will be at least one more QueryInfo in the input stream. // `next_query_info` is allowed to be set only if a method with streaming input (i.e. ExecuteQueryWithStreamInput() or ExecuteQueryWithStreamIO()) is used. bool next_query_info = 16; + + /// Controls how a ClickHouse server will compress query execution results before sending back to the client. + /// If not set the compression settings from the configuration file will be used. + Compression result_compression = 17; } enum LogsLevel { diff --git a/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactoryTest.java b/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactoryTest.java index 09aeb9ec9..e40444a05 100644 --- a/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactoryTest.java +++ b/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactoryTest.java @@ -7,7 +7,7 @@ import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseProtocol; import com.clickhouse.client.ClickHouseRequest; -import com.clickhouse.client.grpc.config.ClickHouseGrpcClientOption; +import com.clickhouse.client.grpc.config.ClickHouseGrpcOption; public class ClickHouseGrpcChannelFactoryTest extends BaseIntegrationTest { @Test(groups = { "integration" }) @@ -18,10 +18,10 @@ public void testGetFactory() throws Exception { Assert.assertTrue(ClickHouseGrpcChannelFactory.getFactory(request.getConfig(), server) instanceof NettyChannelFactoryImpl); Assert.assertTrue(ClickHouseGrpcChannelFactory.getFactory( - request.option(ClickHouseGrpcClientOption.USE_OKHTTP, true).getConfig(), + request.option(ClickHouseGrpcOption.USE_OKHTTP, true).getConfig(), server) instanceof OkHttpChannelFactoryImpl); Assert.assertTrue(ClickHouseGrpcChannelFactory.getFactory( - request.option(ClickHouseGrpcClientOption.USE_OKHTTP, false).getConfig(), + request.option(ClickHouseGrpcOption.USE_OKHTTP, false).getConfig(), server) instanceof NettyChannelFactoryImpl); } } diff --git a/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java b/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java index a75fc3acc..d44e8a9c4 100644 --- a/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java +++ b/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java @@ -41,10 +41,10 @@ import com.clickhouse.client.data.ClickHouseIntegerValue; import com.clickhouse.client.data.ClickHouseIpv4Value; import com.clickhouse.client.data.ClickHouseIpv6Value; -import com.clickhouse.client.exception.ClickHouseException; import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseDataProcessor; import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.client.ClickHouseException; import com.clickhouse.client.ClickHouseFormat; public class ClickHouseGrpcClientTest extends BaseIntegrationTest { @@ -108,7 +108,8 @@ public void testInit() throws Exception { @Test(groups = { "integration" }) public void testOpenCloseConnection() throws Exception { for (int i = 0; i < 100; i++) { - Assert.assertTrue(ClickHouseClient.test(getServer(ClickHouseProtocol.GRPC), 3000)); + Assert.assertTrue(ClickHouseClient.newInstance(ClickHouseProtocol.GRPC) + .ping(getServer(ClickHouseProtocol.GRPC), 3000)); } } @@ -199,8 +200,7 @@ public void testQuery() throws Exception { public void testQueryInSameThread() throws Exception { ClickHouseNode server = getServer(ClickHouseProtocol.GRPC); - try (ClickHouseClient client = ClickHouseClient.builder().addOption(ClickHouseClientOption.ASYNC, false) - .build()) { + try (ClickHouseClient client = ClickHouseClient.builder().option(ClickHouseClientOption.ASYNC, false).build()) { CompletableFuture future = client.connect(server) .format(ClickHouseFormat.TabSeparatedWithNamesAndTypes).query("select 1,2").execute(); // Assert.assertTrue(future instanceof ClickHouseImmediateFuture); @@ -214,7 +214,7 @@ public void testQueryInSameThread() throws Exception { } ClickHouseResponseSummary summary = resp.getSummary(); - Assert.assertEquals(summary.getRows(), 1); + Assert.assertEquals(summary.getStatistics().getRows(), 1); } } } @@ -688,7 +688,7 @@ public void testLoadCsv() throws Exception { ClickHouseResponseSummary summary = ClickHouseClient.load(server, "test_grpc_load_data", ClickHouseFormat.TabSeparated, ClickHouseCompression.NONE, temp.toString()).get(); Assert.assertNotNull(summary); - Assert.assertEquals(summary.getWriteRows(), lines); + Assert.assertEquals(summary.getWrittenRows(), lines); try (ClickHouseClient client = ClickHouseClient.newInstance(server.getProtocol()); ClickHouseResponse resp = client.connect(server) diff --git a/clickhouse-http-client/pom.xml b/clickhouse-http-client/pom.xml index 5a7a60dc2..368647e93 100644 --- a/clickhouse-http-client/pom.xml +++ b/clickhouse-http-client/pom.xml @@ -15,19 +15,41 @@ HTTP client for ClickHouse https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-http-client + + ${project.parent.groupId}.client.internal + + ${project.parent.groupId} clickhouse-client ${revision} + + com.google.code.gson + gson + provided + org.apache.httpcomponents httpclient + provided org.apache.httpcomponents httpmime + provided + + + org.lz4 + lz4-java + + + + + org.apache.tomcat + annotations-api + provided @@ -47,10 +69,68 @@ testcontainers test + + com.github.tomakehurst + wiremock-jre8 + test + org.testng testng test + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade + package + + shade + + + true + true + true + shaded + + + net.jpountz + ${shade.base}.jpountz + + + + + + + + + + *:* + + **/darwin/** + **/linux/** + **/win32/** + META-INF/MANIFEST.MF + META-INF/maven/** + META-INF/native-image/** + META-INF/versions/** + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + \ No newline at end of file diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java new file mode 100644 index 000000000..2067af894 --- /dev/null +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java @@ -0,0 +1,137 @@ +package com.clickhouse.client.http; + +import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import com.clickhouse.client.AbstractClient; +import com.clickhouse.client.ClickHouseCluster; +import com.clickhouse.client.ClickHouseException; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.client.ClickHouseResponse; +import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.client.data.ClickHouseStreamResponse; +import com.clickhouse.client.http.config.ClickHouseHttpOption; +import com.clickhouse.client.logging.Logger; +import com.clickhouse.client.logging.LoggerFactory; + +public class ClickHouseHttpClient extends AbstractClient { + private static final Logger log = LoggerFactory.getLogger(ClickHouseHttpClient.class); + + @Override + protected boolean checkConnection(ClickHouseHttpConnection connection, ClickHouseNode requestServer, + ClickHouseNode currentServer, ClickHouseRequest request) { + // return false to suggest creating a new connection + return connection != null && connection.isReusable() && requestServer.equals(currentServer); + } + + @Override + protected ClickHouseHttpConnection newConnection(ClickHouseHttpConnection connection, ClickHouseNode server, + ClickHouseRequest request) { + if (connection != null && connection.isReusable()) { + closeConnection(connection, false); + } + + try { + return new DefaultHttpConnection(server, request); + } catch (IOException e) { + throw new CompletionException(e); + } + } + + @Override + protected void closeConnection(ClickHouseHttpConnection connection, boolean force) { + try { + connection.close(); + } catch (Exception e) { + log.warn("Failed to close http connection due to: %s", e.getMessage()); + } + } + + protected String buildQueryParams(Map params) { + if (params == null || params.isEmpty()) { + return ""; + } + + StringBuilder builder = new StringBuilder(); + for (Map.Entry entry : params.entrySet()) { + builder.append(ClickHouseHttpConnection.urlEncode(entry.getKey(), StandardCharsets.UTF_8)).append('=') + .append(ClickHouseHttpConnection.urlEncode(entry.getValue(), StandardCharsets.UTF_8)).append('&'); + } + + if (builder.length() > 0) { + builder.setLength(builder.length() - 1); + } + return builder.toString(); + } + + protected ClickHouseResponse postRequest(ClickHouseRequest sealedRequest) throws IOException { + ClickHouseHttpConnection conn = getConnection(sealedRequest); + + List stmts = sealedRequest.getStatements(false); + int size = stmts.size(); + String sql; + if (size == 0) { + throw new IllegalArgumentException("At least one SQL statement is required for execution"); + } else if (size > 1) { + throw new IllegalArgumentException("Expect one SQL statement to execute but we got " + size); + } else { + sql = stmts.get(0); + } + + log.debug("Query: %s", sql); + ClickHouseHttpResponse httpResponse = conn.post(sql, sealedRequest.getInputStream().orElse(null), + sealedRequest.getExternalTables(), null); + return ClickHouseStreamResponse.of(httpResponse.getConfig(sealedRequest), httpResponse, + sealedRequest.getSettings(), null, + httpResponse.summary); + } + + @Override + public boolean accept(ClickHouseProtocol protocol) { + return ClickHouseProtocol.HTTP == protocol || super.accept(protocol); + } + + @Override + public CompletableFuture execute(ClickHouseRequest request) { + // sealedRequest is an immutable copy of the original request + final ClickHouseRequest sealedRequest = request.seal(); + + if (sealedRequest.getConfig().isAsync()) { + return CompletableFuture.supplyAsync(() -> { + try { + return postRequest(sealedRequest); + } catch (IOException e) { + throw new CompletionException(ClickHouseException.of(e, sealedRequest.getServer())); + } + }, getExecutor()); + } else { + try { + return CompletableFuture.completedFuture(postRequest(sealedRequest)); + } catch (IOException e) { + return failedResponse(ClickHouseException.of(e, sealedRequest.getServer())); + } + } + } + + @Override + public final Class getOptionClass() { + return ClickHouseHttpOption.class; + } + + @Override + public boolean ping(ClickHouseNode server, int timeout) { + if (server != null) { + server = ClickHouseCluster.probe(server, timeout); + return getConnection(connect(server)).ping(timeout); + } + + return false; + } +} diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java new file mode 100644 index 000000000..7e166b451 --- /dev/null +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java @@ -0,0 +1,383 @@ +package com.clickhouse.client.http; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Map.Entry; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseCompression; +import com.clickhouse.client.ClickHouseConfig; +import com.clickhouse.client.ClickHouseCredentials; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.data.ClickHouseExternalTable; +import com.clickhouse.client.data.ClickHouseLZ4InputStream; +import com.clickhouse.client.data.ClickHouseLZ4OutputStream; +import com.clickhouse.client.http.config.ClickHouseHttpOption; + +public abstract class ClickHouseHttpConnection implements AutoCloseable { + protected static final int DEFAULT_BUFFER_SIZE = 8192; + + static String urlEncode(String str, Charset charset) { + try { + return URLEncoder.encode(str, StandardCharsets.UTF_8.name()); + } catch (UnsupportedEncodingException e) { + // should not happen + throw new IllegalArgumentException(e); + } + } + + private static StringBuilder appendQueryParameter(StringBuilder builder, String key, String value) { + return builder.append(urlEncode(key, StandardCharsets.UTF_8)).append('=') + .append(urlEncode(value, StandardCharsets.UTF_8)).append('&'); + } + + static String buildQueryParams(ClickHouseRequest request) { + if (request == null) { + return ""; + } + + ClickHouseConfig config = request.getConfig(); + StringBuilder builder = new StringBuilder(); + + // start with custom query parameters first + Map customParams = ClickHouseUtils + .getKeyValuePairs((String) config.getOption(ClickHouseHttpOption.CUSTOM_PARAMS)); + for (Entry cp : customParams.entrySet()) { + appendQueryParameter(builder, cp.getKey(), cp.getValue()); + } + + if (config.isCompressServerResponse()) { + appendQueryParameter(builder, "compress", "1"); + } + if (config.isDecompressClientRequet()) { + appendQueryParameter(builder, "decompress", "1"); + } + + Map settings = request.getSettings(); + List stmts = request.getStatements(false); + String settingKey = "max_execution_time"; + if (config.getMaxExecutionTime() > 0 && !settings.containsKey(settingKey)) { + appendQueryParameter(builder, settingKey, String.valueOf(config.getMaxExecutionTime())); + } + settingKey = "max_result_rows"; + if (config.getMaxResultRows() > 0 && !settings.containsKey(settingKey)) { + appendQueryParameter(builder, settingKey, String.valueOf(config.getMaxExecutionTime())); + appendQueryParameter(builder, "result_overflow_mode", "break"); + } + settingKey = "log_comment"; + if (!stmts.isEmpty() && (boolean) config.getOption(ClickHouseClientOption.LOG_LEADING_COMMENT) + && !settings.containsKey(settingKey)) { + String comment = ClickHouseUtils.getLeadingComment(stmts.get(0)); + if (!comment.isEmpty()) { + appendQueryParameter(builder, settingKey, comment); + } + } + settingKey = "extremes"; + if (!settings.containsKey(settingKey)) { + appendQueryParameter(builder, settingKey, "0"); + } + + Optional optionalValue = request.getSessionId(); + if (optionalValue.isPresent()) { + appendQueryParameter(builder, "session_id", optionalValue.get()); + + if (config.isSessionCheck()) { + appendQueryParameter(builder, "session_check", "1"); + } + if (config.getSessionTimeout() > 0) { + // see default_session_timeout + appendQueryParameter(builder, "session_timeout", String.valueOf(config.getSessionTimeout())); + } + } + + optionalValue = request.getQueryId(); + if (optionalValue.isPresent()) { + appendQueryParameter(builder, "query_id", optionalValue.get()); + } + + for (Map.Entry entry : settings.entrySet()) { + appendQueryParameter(builder, entry.getKey(), String.valueOf(entry.getValue())); + } + + if (builder.length() > 0) { + builder.setLength(builder.length() - 1); + } + return builder.toString(); + } + + static String buildUrl(ClickHouseNode server, ClickHouseRequest request) { + ClickHouseConfig config = request.getConfig(); + + StringBuilder builder = new StringBuilder(); + builder.append(config.isSsl() ? "https" : "http").append("://").append(server.getHost()).append(':') + .append(server.getPort()).append('/'); + String context = (String) config.getOption(ClickHouseHttpOption.WEB_CONTEXT); + if (context != null && !context.isEmpty()) { + char prev = '/'; + for (int i = 0, len = context.length(); i < len; i++) { + char ch = context.charAt(i); + if (ch != '/' || ch != prev) { + builder.append(ch); + } + prev = ch; + } + + if (prev != '/') { + builder.append('/'); + } + } + + String query = buildQueryParams(request); + if (!query.isEmpty()) { + builder.append('?').append(query); + } + + return builder.toString(); + } + + protected final ClickHouseConfig config; + protected final ClickHouseNode server; + protected final Map defaultHeaders; + + protected final String url; + + protected ClickHouseHttpConnection(ClickHouseNode server, ClickHouseRequest request) { + if (server == null || request == null) { + throw new IllegalArgumentException("Non-null server and request are required"); + } + + this.config = request.getConfig(); + this.server = server; + + this.url = buildUrl(server, request); + + Map map = new LinkedHashMap<>(); + // add customer headers + map.putAll(ClickHouseUtils.getKeyValuePairs((String) config.getOption(ClickHouseHttpOption.CUSTOM_HEADERS))); + map.put("Accept", "*/*"); + if (!(boolean) config.getOption(ClickHouseHttpOption.KEEP_ALIVE)) { + map.put("Connection", "Close"); + } + map.put("User-Agent", config.getClientName()); + + ClickHouseCredentials credentials = server.getCredentials(config); + if (credentials.useAccessToken()) { + // TODO check if auth-scheme is available and supported + map.put("Authorization", credentials.getAccessToken()); + } else { + map.put("X-ClickHouse-User", credentials.getUserName()); + if (!ClickHouseChecker.isNullOrEmpty(credentials.getPassword())) { + map.put("X-ClickHouse-Key", credentials.getPassword()); + } + } + map.put("X-ClickHouse-Database", server.getDatabase(config)); + // Also, you can use the ‘default_format’ URL parameter + map.put("X-ClickHouse-Format", config.getFormat().name()); + if (config.isCompressServerResponse()) { + map.put("Accept-Encoding", config.getCompressAlgorithmForServerResponse().encoding()); + } + if (config.isDecompressClientRequet()) { + map.put("Content-Encoding", config.getDecompressAlgorithmForClientRequest().encoding()); + } + + this.defaultHeaders = Collections.unmodifiableMap(map); + } + + protected String getBaseUrl() { + String baseUrl; + int index = url.indexOf('?'); + if (index > 0) { + baseUrl = url.substring(0, index); + } else { + baseUrl = url; + } + + return baseUrl; + } + + protected OutputStream getRequestOutputStream(OutputStream out) throws IOException { + if (!config.isDecompressClientRequet()) { + return out; + } + + // TODO support more algorithms + ClickHouseCompression algorithm = config.getDecompressAlgorithmForClientRequest(); + switch (algorithm) { + case GZIP: + out = new GZIPOutputStream(out, (int) config.getOption(ClickHouseClientOption.MAX_COMPRESS_BLOCK_SIZE)); + break; + case LZ4: + out = new ClickHouseLZ4OutputStream(out, + (int) config.getOption(ClickHouseClientOption.MAX_COMPRESS_BLOCK_SIZE)); + break; + default: + throw new UnsupportedOperationException("Unsupported compression algorithm: " + algorithm); + } + return out; + } + + protected InputStream getResponseInputStream(InputStream in) throws IOException { + if (!config.isCompressServerResponse()) { + return in; + } + + // TODO support more algorithms + ClickHouseCompression algorithm = config.getCompressAlgorithmForServerResponse(); + switch (algorithm) { + case GZIP: + in = new GZIPInputStream(in); + break; + case LZ4: + in = new ClickHouseLZ4InputStream(in); + break; + default: + throw new UnsupportedOperationException("Unsupported compression algorithm: " + algorithm); + } + return in; + } + + protected abstract String getResponseHeader(String header, String defaultValue); + + /** + * Creates a merged map. + * + * @param requestHeaders request headers + * @return + */ + protected Map mergeHeaders(Map requestHeaders) { + if (requestHeaders == null || requestHeaders.isEmpty()) { + return defaultHeaders; + } + + Map merged = new LinkedHashMap<>(); + merged.putAll(defaultHeaders); + for (Entry header : requestHeaders.entrySet()) { + if (header.getValue() == null) { + merged.remove(header.getKey()); + } else { + merged.put(header.getKey(), header.getValue()); + } + } + return merged; + } + + /** + * Pipes data from input stream to output stream. Input stream will be closed + * but output stream will remain open. + * + * @param input non-null input stream, which will be closed + * @param output non-null output stream, which will remain open + * @param bufferSize buffer size, zero or negative number will be treated as + * {@link #DEFAULT_BUFFER_SIZE} + * @throws IOException when error occured reading from input stream or writing + * data to output stream + */ + protected void pipe(InputStream input, OutputStream output, int bufferSize) throws IOException { + if (bufferSize <= 0) { + bufferSize = DEFAULT_BUFFER_SIZE; + } + + byte[] bytes = new byte[bufferSize]; + int counter = 0; + try { + while ((counter = input.read(bytes, 0, bufferSize)) >= 0) { + output.write(bytes, 0, counter); + } + input.close(); + input = null; + } finally { + if (input != null) { + try { + input.close(); + } catch (Exception e) { + // ignore + } + } + } + } + + /** + * Posts query and data to server. + * + * @param query non-blank query + * @param data optionally input stream for batch updating + * @param tables optionally external tables for query + * @param headers optionally request headers + * @return response + * @throws IOException when error occured posting request and/or server failed + * to respond + */ + protected abstract ClickHouseHttpResponse post(String query, InputStream data, List tables, + Map headers) throws IOException; + + /** + * Checks whether the connection is reusable or not. This method will be called + * in + * {@link ClickHouseHttpClient#checkConnection(ClickHouseHttpConnection, ClickHouseNode, ClickHouseNode, ClickHouseRequest)} + * for making a decision of whether to create a new connection. In addition to + * that, if a connection is NOT reusable, it will be closed right after + * corresponding ClickHouseResponse is closed. + * + * @return true if it's reusable; false otherwise + */ + protected boolean isReusable() { + return true; + } + + /** + * Sends a request to {@code /ping} for liveness detection. + * + * @param timeout timeout in millisecond + * @return true if server responded {@code Ok.}; false otherwise + */ + public abstract boolean ping(int timeout); + + public ClickHouseHttpResponse update(String query) throws IOException { + return post(query, null, null, null); + } + + public ClickHouseHttpResponse update(String query, Map headers) throws IOException { + return post(query, null, null, headers); + } + + public ClickHouseHttpResponse update(String query, InputStream data) throws IOException { + return post(query, data, null, null); + } + + public ClickHouseHttpResponse update(String query, InputStream data, Map headers) + throws IOException { + return post(query, data, null, headers); + } + + public ClickHouseHttpResponse query(String query) throws IOException { + return post(query, null, null, null); + } + + public ClickHouseHttpResponse query(String query, Map headers) throws IOException { + return post(query, null, null, headers); + } + + public ClickHouseHttpResponse query(String query, List tables) throws IOException { + return post(query, null, tables, null); + } + + public ClickHouseHttpResponse query(String query, List tables, Map headers) + throws IOException { + return post(query, null, tables, headers); + } +} diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpResponse.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpResponse.java new file mode 100644 index 000000000..373119b7f --- /dev/null +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpResponse.java @@ -0,0 +1,138 @@ +package com.clickhouse.client.http; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.TimeZone; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseConfig; +import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.client.ClickHouseResponseSummary; +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.config.ClickHouseOption; + +public class ClickHouseHttpResponse extends InputStream { + private static long getLongValue(Map map, String key) { + String value = map.get(key); + if (value != null) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + // ignore error + } + } + return 0L; + } + + private final ClickHouseHttpConnection connection; + private final InputStream input; + + protected final String serverDisplayName; + protected final String queryId; + protected final ClickHouseFormat format; + protected final TimeZone timeZone; + + protected final ClickHouseResponseSummary summary; + + protected ClickHouseConfig getConfig(ClickHouseRequest request) { + ClickHouseConfig config = request.getConfig(); + if (format != null && format != config.getFormat()) { + Map options = new HashMap<>(); + options.putAll(config.getAllOptions()); + options.put(ClickHouseClientOption.FORMAT, format); + config = new ClickHouseConfig(options, config.getDefaultCredentials(), config.getNodeSelector(), + config.getMetricRegistry()); + } + return config; + } + + public ClickHouseHttpResponse(ClickHouseHttpConnection connection, InputStream input) { + if (connection == null || input == null) { + throw new IllegalArgumentException("Non-null connection and input stream are required"); + } + + this.connection = connection; + this.input = input; + + this.serverDisplayName = connection.getResponseHeader("X-ClickHouse-Server-Display-Name", + connection.server.getHost()); + // queryId, format and timeZone are only available for queries + this.queryId = connection.getResponseHeader("X-ClickHouse-Query-Id", ""); + // {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} + Map map = (Map) ClickHouseUtils + .parseJson(connection.getResponseHeader("X-ClickHouse-Summary", "{}")); + // discard those X-ClickHouse-Progress headers + this.summary = new ClickHouseResponseSummary( + new ClickHouseResponseSummary.Progress(getLongValue(map, "read_rows"), getLongValue(map, "read_bytes"), + getLongValue(map, "total_rows_to_read"), getLongValue(map, "written_rows"), + getLongValue(map, "written_bytes")), + null); + + if (ClickHouseChecker.isNullOrEmpty(this.queryId)) { + this.format = connection.config.getFormat(); + this.timeZone = connection.config.getServerTimeZone(); + // better to close input stream since there's no response to read? + // input.close(); + } else { + String value = connection.getResponseHeader("X-ClickHouse-Format", ""); + this.format = !ClickHouseChecker.isNullOrEmpty(value) ? ClickHouseFormat.valueOf(value) + : connection.config.getFormat(); + value = connection.getResponseHeader("X-ClickHouse-Timezone", ""); + this.timeZone = !ClickHouseChecker.isNullOrEmpty(value) ? TimeZone.getTimeZone(value) + : connection.config.getServerTimeZone(); + } + } + + @Override + public int read() throws IOException { + return input.read(); + } + + @Override + public int available() throws IOException { + return input.available(); + } + + @Override + public void close() throws IOException { + IOException error = null; + + try { + input.close(); + } catch (IOException e) { + error = e; + } + + if (!connection.isReusable()) { + try { + connection.close(); + } catch (Exception e) { + // ignore + } + } + + if (error != null) { + throw error; + } + } + + @Override + public boolean markSupported() { + return false; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return input.read(b, off, len); + } + + @Override + public long skip(long n) throws IOException { + return input.skip(n); + } +} diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/DefaultHttpConnection.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/DefaultHttpConnection.java new file mode 100644 index 000000000..905d0611b --- /dev/null +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/DefaultHttpConnection.java @@ -0,0 +1,215 @@ +package com.clickhouse.client.http; + +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.client.ClickHouseSslContextProvider; +import com.clickhouse.client.config.ClickHouseSslMode; +import com.clickhouse.client.data.ClickHouseExternalTable; +import com.clickhouse.client.http.config.ClickHouseHttpOption; +import com.clickhouse.client.logging.Logger; +import com.clickhouse.client.logging.LoggerFactory; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.Map.Entry; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; + +public class DefaultHttpConnection extends ClickHouseHttpConnection { + private static final Logger log = LoggerFactory.getLogger(DefaultHttpConnection.class); + + private final HttpURLConnection conn; + + private HttpURLConnection newConnection(String url, boolean post) throws IOException { + HttpURLConnection newConn = (HttpURLConnection) new URL(url).openConnection(); + + if ((newConn instanceof HttpsURLConnection) && config.isSsl()) { + HttpsURLConnection secureConn = (HttpsURLConnection) newConn; + SSLContext sslContext = ClickHouseSslContextProvider.getProvider().getSslContext(SSLContext.class, config) + .orElse(null); + HostnameVerifier verifier = config.getSslMode() == ClickHouseSslMode.STRICT + ? HttpsURLConnection.getDefaultHostnameVerifier() + : (hostname, session) -> true; + + secureConn.setHostnameVerifier(verifier); + secureConn.setSSLSocketFactory(sslContext.getSocketFactory()); + } + + if (post) { + newConn.setInstanceFollowRedirects(true); + newConn.setRequestMethod("POST"); + } + newConn.setUseCaches(false); + newConn.setAllowUserInteraction(false); + newConn.setDoInput(true); + newConn.setDoOutput(true); + newConn.setConnectTimeout(config.getConnectionTimeout()); + newConn.setReadTimeout(config.getSocketTimeout()); + + return newConn; + } + + private void setHeaders(HttpURLConnection conn, Map headers) { + headers = mergeHeaders(headers); + + if (headers == null || headers.isEmpty()) { + return; + } + + for (Entry header : headers.entrySet()) { + conn.setRequestProperty(header.getKey(), header.getValue()); + } + } + + private void checkResponse(HttpURLConnection conn) throws IOException { + if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) { + // TODO get exception from response header, for example: + // X-ClickHouse-Exception-Code: 47 + StringBuilder builder = new StringBuilder(); + try (Reader reader = new BufferedReader( + new InputStreamReader(getResponseInputStream(conn.getErrorStream()), StandardCharsets.UTF_8))) { + int c = 0; + while ((c = reader.read()) != -1) { + builder.append((char) c); + } + } catch (IOException e) { + log.warn("Error while reading error message", e); + } + + throw new IOException(builder.toString()); + } + } + + protected DefaultHttpConnection(ClickHouseNode server, ClickHouseRequest request) throws IOException { + super(server, request); + + conn = newConnection(url, true); + } + + @Override + protected boolean isReusable() { + return false; + } + + @Override + protected String getResponseHeader(String header, String defaultValue) { + String value = conn.getHeaderField(header); + return value != null ? value : defaultValue; + } + + @Override + protected ClickHouseHttpResponse post(String sql, InputStream data, List tables, + Map headers) throws IOException { + String boundary = null; + if (tables != null && !tables.isEmpty()) { + boundary = UUID.randomUUID().toString(); + conn.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary); + } else { + conn.setRequestProperty("Content-Type", "text/plain; charset=UTF-8"); + } + setHeaders(conn, headers); + + try (OutputStream out = getRequestOutputStream(conn.getOutputStream()); + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) { + if (boundary != null) { + String line = "\r\n--" + boundary + "\r\n"; + writer.write(line); + writer.write("Content-Disposition: form-data; name=\"query\"\r\n\r\n"); + writer.write(sql); + + for (ClickHouseExternalTable t : tables) { + String tableName = t.getName(); + StringBuilder builder = new StringBuilder(); + builder.append(line).append("Content-Disposition: form-data; name=\"").append(tableName) + .append("_format\"\r\n\r\n").append(t.getFormat().name()); + builder.append(line).append("Content-Disposition: form-data; name=\"").append(tableName) + .append("_structure\"\r\n\r\n").append(t.getStructure()); + builder.append(line).append("Content-Disposition: form-data; name=\"").append(tableName) + .append("\"; filename=\"").append(tableName).append("\"\r\n") + .append("Content-Type: application/octet-stream\r\n") + .append("Content-Transfer-Encoding: binary\r\n\r\n"); + writer.write(builder.toString()); + writer.flush(); + + pipe(t.getContent(), out, DEFAULT_BUFFER_SIZE); + } + + writer.write("\r\n--" + boundary + "--\r\n"); + writer.flush(); + } else { + writer.write(sql); + writer.flush(); + + if (data != null && data.available() > 0) { + // append \n + if (sql.charAt(sql.length() - 1) != '\n') { + out.write(10); + } + + pipe(data, out, DEFAULT_BUFFER_SIZE); + } + } + } + + checkResponse(conn); + + // X-ClickHouse-Server-Display-Name: xxx + // X-ClickHouse-Query-Id: xxx + // X-ClickHouse-Format: RowBinaryWithNamesAndTypes + // X-ClickHouse-Timezone: UTC + // X-ClickHouse-Summary: + // {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} + + return new ClickHouseHttpResponse(this, getResponseInputStream(conn.getInputStream())); + } + + @Override + public boolean ping(int timeout) { + String response = (String) config.getOption(ClickHouseHttpOption.DEFAULT_RESPONSE); + HttpURLConnection c = null; + try { + c = newConnection(getBaseUrl() + "ping", false); + c.setConnectTimeout(timeout); + c.setReadTimeout(timeout); + + checkResponse(c); + + int size = 12; + try (ByteArrayOutputStream out = new ByteArrayOutputStream(size)) { + pipe(c.getInputStream(), out, size); + + c.disconnect(); + c = null; + return response.equals(new String(out.toByteArray(), StandardCharsets.UTF_8)); + } + } catch (IOException e) { + log.debug("Failed to ping server: ", e.getMessage()); + } finally { + if (c != null) { + c.disconnect(); + } + } + + return false; + } + + @Override + public void close() { + conn.disconnect(); + } +} diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java new file mode 100644 index 000000000..cd1ce8500 --- /dev/null +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java @@ -0,0 +1,76 @@ +package com.clickhouse.client.http.config; + +import java.io.Serializable; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.config.ClickHouseOption; + +/** + * gRPC client options. + */ +public enum ClickHouseHttpOption implements ClickHouseOption { + /** + * Custom HTTP headers. + */ + CUSTOM_HEADERS("custom_http_headers", "", "Custom HTTP headers."), + /** + * Custom HTTP query parameters. + */ + CUSTOM_PARAMS("custom_http_params", "", "Custom HTTP query parameters."), + /** + * Default server response. + */ + DEFAULT_RESPONSE("http_server_default_response", "Ok.\n", + "Default server response, which is used for validating connection."), + /** + * Whether to enable keep-alive or not. + */ + KEEP_ALIVE("http_keep_alive", true, "Whether to use keep-alive or not"), + /** + * Whether to receive information about the progress of a query in response + * headers. + */ + RECEIVE_QUERY_PROGRESS("receive_query_progress", true, + "Whether to receive information about the progress of a query in response headers."), + // SEND_PROGRESS("send_progress_in_http_headers", false, + // "Enables or disables X-ClickHouse-Progress HTTP response headers in + // clickhouse-server responses."), + // SEND_PROGRESS_INTERVAL("http_headers_progress_interval_ms", 3000, ""), + // WAIT_END_OF_QUERY("wait_end_of_query", false, ""), + /** + * Flow control window. + */ + WEB_CONTEXT("web_context", "/", "Web context."); + + private final String key; + private final Serializable defaultValue; + private final Class clazz; + private final String description; + + ClickHouseHttpOption(String key, T defaultValue, String description) { + this.key = ClickHouseChecker.nonNull(key, "key"); + this.defaultValue = ClickHouseChecker.nonNull(defaultValue, "defaultValue"); + this.clazz = defaultValue.getClass(); + this.description = ClickHouseChecker.nonNull(description, "description"); + } + + @Override + public Serializable getDefaultValue() { + return defaultValue; + } + + @Override + public String getDescription() { + return description; + } + + @Override + public String getKey() { + return key; + } + + @Override + public Class getValueType() { + return clazz; + } +} diff --git a/clickhouse-http-client/src/main/java9/module-info.java b/clickhouse-http-client/src/main/java9/module-info.java new file mode 100644 index 000000000..d1f979b0d --- /dev/null +++ b/clickhouse-http-client/src/main/java9/module-info.java @@ -0,0 +1,10 @@ +module com.clickhouse.client.http { + exports com.clickhouse.client.http; + exports com.clickhouse.client.http.config; + + provides com.clickhouse.client.ClickHouseClient with com.clickhouse.client.http.ClickHouseHttpClient; + + requires static com.google.gson; + + requires transitive com.clickhouse.client; +} diff --git a/clickhouse-http-client/src/main/resources/META-INF/services/com.clickhouse.client.ClickHouseClient b/clickhouse-http-client/src/main/resources/META-INF/services/com.clickhouse.client.ClickHouseClient new file mode 100644 index 000000000..1c72e6594 --- /dev/null +++ b/clickhouse-http-client/src/main/resources/META-INF/services/com.clickhouse.client.ClickHouseClient @@ -0,0 +1 @@ +com.clickhouse.client.http.ClickHouseHttpClient diff --git a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java new file mode 100644 index 000000000..17eb4f37f --- /dev/null +++ b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java @@ -0,0 +1,227 @@ +package com.clickhouse.client.http; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import com.clickhouse.client.BaseIntegrationTest; +import com.clickhouse.client.ClickHouseClient; +import com.clickhouse.client.ClickHouseCredentials; +import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseParameterizedQuery; +import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseRecord; +import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.client.ClickHouseResponse; +import com.clickhouse.client.ClickHouseResponseSummary; +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.data.ClickHouseExternalTable; +import com.clickhouse.client.data.ClickHouseStringValue; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseHttpClientTest extends BaseIntegrationTest { + @Test(groups = { "integration" }) + public void testPing() throws Exception { + ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); + + try (ClickHouseClient client = ClickHouseClient.newInstance()) { + Assert.assertTrue(client.ping(server, 1000)); + } + } + + @Test(groups = { "integration" }) + public void testMutation() throws Exception { + ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); + ClickHouseClient.send(server, "drop table if exists test_data_load", + "create table test_data_load(a String, b Nullable(Int64))engine=Memory").get(); + try (ClickHouseClient client = ClickHouseClient.newInstance(); + ClickHouseResponse response = client.connect(server).set("send_progress_in_http_headers", 1) + .query("insert into test_data_load select toString(number), number from numbers(1)").execute() + .get()) { + ClickHouseResponseSummary summary = response.getSummary(); + Assert.assertEquals(summary.getWrittenRows(), 1); + } + } + + @Test(groups = { "integration" }) + public void testMultipleQueries() throws Exception { + ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); + try (ClickHouseClient client = ClickHouseClient.newInstance()) { + ClickHouseRequest req = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes); + + ClickHouseResponse queryResp = req.copy().query("select 1").execute().get(); + + try (ClickHouseResponse resp = req.copy().query("select 2").execute().get()) { + } + + for (ClickHouseRecord r : queryResp.records()) { + continue; + } + } + } + + @Test(groups = { "integration" }) + public void testExternalTableAsParameter() throws Exception { + ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); + try (ClickHouseClient client = ClickHouseClient.newInstance(); + ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .query("select toString(number) as query_id from numbers(100) where query_id not in (select query_id from ext_table) limit 10") + .external(ClickHouseExternalTable.builder().name("ext_table") + .columns("query_id String, a_num Nullable(Int32)").format(ClickHouseFormat.CSV) + .content(new ByteArrayInputStream("\"1,2,3\",\\N\n2,333".getBytes())).build()) + .execute().get()) { + for (ClickHouseRecord r : resp.records()) { + Assert.assertNotNull(r); + } + } + } + + @Test(groups = { "integration" }) + public void testInsertWithInputFunction() throws Exception { + ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); + ClickHouseClient.send(server, "drop table if exists test_input_function", + "create table test_input_function(name String, value Nullable(Int32))engine=Memory").get(); + + try (ClickHouseClient client = ClickHouseClient.newInstance()) { + // default format ClickHouseFormat.TabSeparated + ClickHouseRequest req = client.connect(server); + try (ClickHouseResponse resp = req.write().query( + "insert into test_input_function select col2, col3 from input('col1 UInt8, col2 String, col3 Int32')") + .data(new ByteArrayInputStream("1\t2\t33\n2\t3\t333".getBytes())).execute().get()) { + + } + + List values = new ArrayList<>(); + try (ClickHouseResponse resp = req.query("select * from test_input_function").execute().get()) { + for (ClickHouseRecord r : resp.records()) { + values.add(new Object[] { r.getValue(0).asObject() }); + } + } + Assert.assertEquals(values.toArray(new Object[0][]), + new Object[][] { new Object[] { "2\t33" }, new Object[] { "3\t333" } }); + } + } + + @Test(groups = { "integration" }) + public void testLogComment() throws Exception { + ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); + String uuid = UUID.randomUUID().toString(); + try (ClickHouseClient client = ClickHouseClient.newInstance()) { + ClickHouseRequest request = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes); + try (ClickHouseResponse resp = request + .option(ClickHouseClientOption.LOG_LEADING_COMMENT, true) + .query("-- select something\r\nselect 1", uuid).execute().get()) { + } + + try (ClickHouseResponse resp = request + .option(ClickHouseClientOption.LOG_LEADING_COMMENT, true) + .query("SYSTEM FLUSH LOGS", uuid).execute().get()) { + } + + try (ClickHouseResponse resp = request + .option(ClickHouseClientOption.LOG_LEADING_COMMENT, true) + .query(ClickHouseParameterizedQuery + .of("select log_comment from system.query_log where query_id = :qid")) + .params(ClickHouseStringValue.of(uuid)).execute().get()) { + int counter = 0; + for (ClickHouseRecord r : resp.records()) { + Assert.assertEquals(r.getValue(0).asString(), "select something"); + counter++; + } + Assert.assertEquals(counter, 2); + } + } + } + + @Test(groups = "integration") + public void testQueryWithMultipleExternalTables() throws Exception { + ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); + + int tables = 30; + int rows = 10; + try (ClickHouseClient client = ClickHouseClient.builder().build()) { + try (ClickHouseResponse resp = client.connect(server).query("drop table if exists test_ext_data_query") + .execute().get()) { + } + + String ddl = "create table test_ext_data_query (\n" + " Cb String,\n" + " CREATETIME DateTime64(3),\n" + + " TIMESTAMP UInt64,\n" + " Cc String,\n" + " Ca1 UInt64,\n" + " Ca2 UInt64,\n" + + " Ca3 UInt64\n" + ") engine = MergeTree()\n" + "PARTITION BY toYYYYMMDD(CREATETIME)\n" + + "ORDER BY (Cb, CREATETIME, Cc);"; + try (ClickHouseResponse resp = client.connect(server).query(ddl).execute().get()) { + } + } + + String template = "avgIf(Ca1, Cb in L%1$d) as avgCa1%2$d, sumIf(Ca1, Cb in L%1$d) as sumCa1%2$d, minIf(Ca1, Cb in L%1$d) as minCa1%2$d, maxIf(Ca1, Cb in L%1$d) as maxCa1%2$d, anyIf(Ca1, Cb in L%1$d) as anyCa1%2$d, avgIf(Ca2, Cb in L%1$d) as avgCa2%2$d, sumIf(Ca2, Cb in L%1$d) as sumCa2%2$d, minIf(Ca2, Cb in L%1$d) as minCa2%2$d, maxIf(Ca2, Cb in L%1$d) as maxCa2%2$d, anyIf(Ca2, Cb in L%1$d) as anyCa2%2$d, avgIf(Ca3, Cb in L%1$d) as avgCa3%2$d, sumIf(Ca3, Cb in L%1$d) as sumCa3%2$d, minIf(Ca3, Cb in L%1$d) as minCa3%2$d, maxIf(Ca3, Cb in L%1$d) as maxCa3%2$d, anyIf(Ca3, Cb in L%1$d) as anyCa3%2$d"; + StringBuilder sql = new StringBuilder().append("select "); + List extTableList = new ArrayList<>(tables); + for (int i = 0; i < tables; i++) { + sql.append(ClickHouseUtils.format(template, i, i + 1)).append(','); + List valueList = new ArrayList<>(rows); + for (int j = i, size = i + rows; j < size; j++) { + valueList.add(String.valueOf(j)); + } + String dnExtString = String.join("\n", valueList); + InputStream inputStream = new ByteArrayInputStream(dnExtString.getBytes(Charset.forName("UTF-8"))); + ClickHouseExternalTable extTable = ClickHouseExternalTable.builder().name("L" + i).content(inputStream) + .addColumn("Cb", "String").build(); + extTableList.add(extTable); + } + + if (tables > 0) { + sql.deleteCharAt(sql.length() - 1); + } else { + sql.append('*'); + } + sql.append( + " from test_ext_data_query where TIMESTAMP >= 1625796480 and TIMESTAMP < 1625796540 and Cc = 'eth0'"); + + try (ClickHouseClient client = ClickHouseClient.builder().build(); + ClickHouseResponse resp = client.connect(server).query(sql.toString()) + .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).external(extTableList).execute().get()) { + Assert.assertNotNull(resp.getColumns()); + Assert.assertTrue(tables <= 0 || resp.records().iterator().hasNext()); + } + } + + @Test(groups = { "integration" }) + public void testPost() throws Exception { + ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); + + try (ClickHouseClient client = ClickHouseClient.builder() + .defaultCredentials(ClickHouseCredentials.fromUserAndPassword("foo", "bar")).build()) { + // why no detailed error message for this: "select 1，2" + try (ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .query("select 1,2").execute().get()) { + int count = 0; + for (ClickHouseRecord r : resp.records()) { + Assert.assertEquals(r.getValue(0).asInteger(), 1); + Assert.assertEquals(r.getValue(1).asInteger(), 2); + count++; + } + + Assert.assertEquals(count, 1); + } + + // reuse connection + try (ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .query("select 3,4").execute().get()) { + int count = 0; + for (ClickHouseRecord r : resp.records()) { + Assert.assertEquals(r.getValue(0).asInteger(), 3); + Assert.assertEquals(r.getValue(1).asInteger(), 4); + count++; + } + + Assert.assertEquals(count, 1); + } + } + } +} diff --git a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpConnectionTest.java b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpConnectionTest.java new file mode 100644 index 000000000..e72fb6c2a --- /dev/null +++ b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpConnectionTest.java @@ -0,0 +1,81 @@ +package com.clickhouse.client.http; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Map; + +import com.clickhouse.client.ClickHouseClient; +import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.client.data.ClickHouseExternalTable; +import com.clickhouse.client.http.config.ClickHouseHttpOption; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseHttpConnectionTest { + static class SimpleHttpConnection extends ClickHouseHttpConnection { + protected SimpleHttpConnection(ClickHouseNode server, ClickHouseRequest request) { + super(server, request); + } + + @Override + protected ClickHouseHttpResponse post(String query, InputStream data, List tables, + Map headers) throws IOException { + return null; + } + + @Override + public boolean ping(int timeout) { + return false; + } + + @Override + public void close() throws Exception { + } + + @Override + protected String getResponseHeader(String header, String defaultValue) { + return defaultValue; + } + } + + @Test(groups = { "unit" }) + public void testDefaultHeaders() throws Exception { + ClickHouseNode server = ClickHouseNode.builder().build(); + ClickHouseRequest request = ClickHouseClient.newInstance().connect(server); + SimpleHttpConnection sc = new SimpleHttpConnection(server, request); + Assert.assertTrue(!sc.defaultHeaders.isEmpty()); + Assert.assertEquals(sc.defaultHeaders, sc.mergeHeaders(null)); + + sc = new SimpleHttpConnection(server, request.format(ClickHouseFormat.ArrowStream)); + Assert.assertTrue(!sc.defaultHeaders.isEmpty()); + Assert.assertEquals(sc.defaultHeaders, sc.mergeHeaders(null)); + } + + @Test(groups = { "unit" }) + public void testBuildUrl() throws Exception { + ClickHouseNode server = ClickHouseNode.builder().build(); + ClickHouseRequest request = ClickHouseClient.newInstance().connect(server); + Assert.assertEquals(ClickHouseHttpConnection.buildUrl(server, request), + "http://localhost:8123/?compress=1&extremes=0"); + + Assert.assertEquals( + ClickHouseHttpConnection.buildUrl(server, request.option(ClickHouseHttpOption.WEB_CONTEXT, "")), + "http://localhost:8123/?compress=1&extremes=0"); + Assert.assertEquals( + ClickHouseHttpConnection.buildUrl(server, request.option(ClickHouseHttpOption.WEB_CONTEXT, "/")), + "http://localhost:8123/?compress=1&extremes=0"); + Assert.assertEquals( + ClickHouseHttpConnection.buildUrl(server, request.option(ClickHouseHttpOption.WEB_CONTEXT, ".")), + "http://localhost:8123/./?compress=1&extremes=0"); + Assert.assertEquals( + ClickHouseHttpConnection.buildUrl(server, request.option(ClickHouseHttpOption.WEB_CONTEXT, "./")), + "http://localhost:8123/./?compress=1&extremes=0"); + Assert.assertEquals( + ClickHouseHttpConnection.buildUrl(server, request.option(ClickHouseHttpOption.WEB_CONTEXT, "///.//")), + "http://localhost:8123/./?compress=1&extremes=0"); + } +} diff --git a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/DefaultHttpConnectionTest.java b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/DefaultHttpConnectionTest.java new file mode 100644 index 000000000..861bf745a --- /dev/null +++ b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/DefaultHttpConnectionTest.java @@ -0,0 +1,28 @@ +package com.clickhouse.client.http; + +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; + +import com.clickhouse.client.BaseIntegrationTest; +import com.clickhouse.client.ClickHouseClient; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseRequest; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class DefaultHttpConnectionTest extends BaseIntegrationTest { + @Test(groups = { "integration" }) + public void testConnectionReuse() throws Exception { + ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); + + try (ClickHouseClient client = ClickHouseClient.newInstance()) { + ClickHouseRequest req = client.connect(server); + + DefaultHttpConnection conn = new DefaultHttpConnection(server, req); + } + } +} diff --git a/clickhouse-http-client/src/test/resources/log4j.properties b/clickhouse-http-client/src/test/resources/log4j.properties new file mode 100644 index 000000000..204c71e96 --- /dev/null +++ b/clickhouse-http-client/src/test/resources/log4j.properties @@ -0,0 +1,5 @@ +log4j.rootLogger=INFO, STDOUT +log4j.category.com.clickhouse.client=DEBUG +log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender +log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout +log4j.appender.STDOUT.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.sss} [%t] [%-5p] {%c{1}:%L} - %m%n diff --git a/clickhouse-jdbc/legacy.xml b/clickhouse-jdbc/legacy.xml new file mode 100644 index 000000000..5442b8402 --- /dev/null +++ b/clickhouse-jdbc/legacy.xml @@ -0,0 +1,262 @@ + + 4.0.0 + + + com.clickhouse + clickhouse-java + ${revision} + + + ru.yandex.clickhouse + clickhouse-jdbc + ${revision} + jar + + ${project.artifactId} + JDBC driver for ClickHouse + https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-jdbc + + + + serebrserg + Sergey Serebryanik + serebrserg@yandex-team.ru + + + jkee + Viktor Tarnavsky + jkee@yandex-team.ru + + + orantius + Yuriy Galitskiy + orantius@yandex-team.ru + + + krash + Alexandr Krasheninnikov + krash3@gmail.com + + + zhicwu + Zhichun Wu + zhicwu@gmail.com + + + + + 4.5.13 + 4.1.4 + ru.yandex.clickhouse.jdbc.internal + JDBC + 4.2 + + + + + ${project.parent.groupId} + clickhouse-http-client + ${revision} + + + ${project.parent.groupId} + org.roaringbitmap + ${repackaged.version} + provided + + + * + * + + + + + com.google.code.gson + gson + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + org.apache.httpcomponents + httpmime + ${httpclient.version} + + + org.lz4 + lz4-java + + + + ${project.parent.groupId} + clickhouse-client + ${revision} + test-jar + test + + + org.slf4j + slf4j-log4j12 + test + + + org.mockito + mockito-core + test + + + com.github.tomakehurst + wiremock-jre8 + test + + + org.testcontainers + testcontainers + test + + + org.testng + testng + test + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + com.helger.maven + ph-javacc-maven-plugin + ${javacc-plugin.version} + + + jjc + generate-sources + + javacc + + + 1.8 + true + com.clickhouse.jdbc.parser + src/main/javacc + src/main/java + + + + + + io.github.git-commit-id + git-commit-id-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + true + true + true + + + ${spec.title} + ${spec.version} + ${project.groupId} + ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + jar-with-dependencies + + + + true + true + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade + package + + shade + + + true + true + true + shaded + + + com.google.gson + ${shade.base}.gson + + + org.apache + ${shade.base}.apache + + + net.jpountz + ${shade.base}.jpountz + + + + + + + + + + ${spec.title} + ${spec.version} + ${project.name} + ${project.groupId} + ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) + + + + + + *:* + + mozilla/** + **/darwin/** + **/linux/** + **/win32/** + META-INF/DEPENDENCIES + META-INF/MANIFEST.MF + META-INF/maven/** + META-INF/native-image/** + META-INF/versions/** + META-INF/*.xml + + + + + + + + + + \ No newline at end of file diff --git a/clickhouse-jdbc/pom.xml b/clickhouse-jdbc/pom.xml index d5878b9bf..9a2957f9d 100644 --- a/clickhouse-jdbc/pom.xml +++ b/clickhouse-jdbc/pom.xml @@ -7,7 +7,6 @@ ${revision} - ru.yandex.clickhouse clickhouse-jdbc ${revision} jar @@ -16,38 +15,10 @@ JDBC driver for ClickHouse https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-jdbc - - - serebrserg - Sergey Serebryanik - serebrserg@yandex-team.ru - - - jkee - Viktor Tarnavsky - jkee@yandex-team.ru - - - orantius - Yuriy Galitskiy - orantius@yandex-team.ru - - - krash - Alexandr Krasheninnikov - krash3@gmail.com - - - zhicwu - Zhichun Wu - zhicwu@gmail.com - - - 4.5.13 4.1.4 - ru.yandex.clickhouse.jdbc.internal + com.clickhouse.jdbc.internal JDBC 4.2 @@ -57,6 +28,25 @@ ${project.parent.groupId} clickhouse-client ${revision} + provided + + + ${project.parent.groupId} + clickhouse-grpc-client + netty + ${revision} + + + * + * + + + + + ${project.parent.groupId} + clickhouse-http-client + shaded + ${revision} * @@ -127,6 +117,21 @@ testng test + + mysql + mysql-connector-java + test + + + org.mariadb.jdbc + mariadb-java-client + test + + + org.postgresql + postgresql + test + @@ -153,13 +158,17 @@ 1.8 true - ru.yandex.clickhouse.jdbc.parser + com.clickhouse.jdbc.parser src/main/javacc src/main/java + + io.github.git-commit-id + git-commit-id-maven-plugin + org.apache.maven.plugins maven-jar-plugin @@ -174,6 +183,7 @@ ${spec.title} ${spec.version} ${project.groupId} + ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) @@ -196,64 +206,267 @@ org.apache.maven.plugins maven-shade-plugin - - true - true - true - shaded - - - com.google.gson - ${shade.base}.gson - - - org.apache - ${shade.base}.apache - - - net.jpountz - ${shade.base}.jpountz - - - - - - - - - - ${spec.title} - ${spec.version} - ${project.name} - ${project.version} - ${project.groupId} - - - - - - *:* - - mozilla/** - **/darwin/** - **/linux/** - **/win32/** - META-INF/maven/** - META-INF/native-image/** - META-INF/versions/** - META-INF/*.xml - - - - + shade + package + + shade + + + true + true + true + shaded + + + com.google.gson + ${shade.base}.gson + + + org.apache + ${shade.base}.apache + + + net.jpountz + ${shade.base}.jpountz + + + + + + + + + + ${spec.title} + ${spec.version} + ${project.name} + ${project.groupId} + ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) + + + + + + ${project.parent.groupId}:clickhouse-grpc-client + + ** + + + + *:* + + mozilla/** + **/darwin/** + **/linux/** + **/win32/** + META-INF/DEPENDENCIES + META-INF/MANIFEST.MF + META-INF/maven/** + META-INF/native-image/** + META-INF/versions/** + META-INF/*.xml + + + + + + + shade-all + package + + shade + + + true + true + true + all + + + com.google.gson + ${shade.base}.gson + + + org.apache + ${shade.base}.apache + + + net.jpountz + ${shade.base}.jpountz + + + + + + + + + + ${spec.title} + ${spec.version} + ${project.name} + ${project.groupId} + ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) + + + + + + *:* + + mozilla/** + **/darwin/** + **/linux/** + **/win32/** + META-INF/DEPENDENCIES + META-INF/MANIFEST.MF + META-INF/maven/** + META-INF/native-image/** + META-INF/versions/** + META-INF/*.xml + + + + + + + shade-grpc + package + + shade + + + true + true + true + grpc + + + com.google.gson + ${shade.base}.gson + + + net.jpountz + ${shade.base}.jpountz + + + + + + + + + + ${spec.title} + ${spec.version} + ${project.name} + ${project.groupId} + ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) + + + + + + ${project.parent.groupId}:clickhouse-http-client + + ** + + + + org.apache.httpcomponents:* + + ** + + + + *:* + + mozilla/** + **/darwin/** + **/linux/** + **/win32/** + META-INF/DEPENDENCIES + META-INF/MANIFEST.MF + META-INF/maven/** + META-INF/native-image/** + META-INF/versions/** + META-INF/*.xml + + + + + + + shade-http package shade - false + true + true + true + http + + + + + + + + ${spec.title} + ${spec.version} + ${project.name} + ${project.groupId} + ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) + + + + + + ${project.parent.groupId}:clickhouse-grpc-client + + ** + + + + com.google.code.gson:* + + ** + + + + org.lz4:lz4-java + + ** + + + + org.apache.httpcomponents:* + + ** + + + + *:* + + mozilla/** + org/** + ru/** + **/darwin/** + **/linux/** + **/win32/** + META-INF/DEPENDENCIES + META-INF/MANIFEST.MF + META-INF/maven/** + META-INF/native-image/** + META-INF/versions/** + META-INF/*.xml + + + diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/AbstractResultSet.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/AbstractResultSet.java new file mode 100644 index 000000000..d4be1244e --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/AbstractResultSet.java @@ -0,0 +1,712 @@ +package com.clickhouse.jdbc; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.sql.Array; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.Date; +import java.sql.NClob; +import java.sql.Ref; +import java.sql.ResultSet; +import java.sql.RowId; +import java.sql.SQLException; +import java.sql.SQLType; +import java.sql.SQLWarning; +import java.sql.SQLXML; +import java.sql.Time; +import java.sql.Timestamp; + +public abstract class AbstractResultSet extends Wrapper implements ResultSet { + protected void ensureOpen() throws SQLException { + if (isClosed()) { + throw SqlExceptionUtils.clientError("Cannot operate on a closed ResultSet"); + } + } + + @Override + public boolean absolute(int row) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("absolute not implemented"); + } + + @Override + public void afterLast() throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("afterLast not implemented"); + } + + @Override + public void beforeFirst() throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("beforeFirst not implemented"); + } + + @Override + public void cancelRowUpdates() throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("cancelRowUpdates not implemented"); + } + + @Override + public void clearWarnings() throws SQLException { + ensureOpen(); + } + + @Override + public void deleteRow() throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("deleteRow not implemented"); + } + + @Override + public boolean first() throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("first not implemented"); + } + + @Override + public int getConcurrency() throws SQLException { + ensureOpen(); + + return ResultSet.CONCUR_READ_ONLY; + } + + @Override + public int getFetchDirection() throws SQLException { + ensureOpen(); + + return ResultSet.FETCH_FORWARD; + } + + @Override + public int getHoldability() throws SQLException { + ensureOpen(); + + return ResultSet.HOLD_CURSORS_OVER_COMMIT; + } + + @Override + public int getType() throws SQLException { + ensureOpen(); + + return ResultSet.TYPE_FORWARD_ONLY; + } + + @Override + public SQLWarning getWarnings() throws SQLException { + ensureOpen(); + + return null; + } + + @Override + public void insertRow() throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("insertRow not implemented"); + } + + @Override + public boolean last() throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("last not implemented"); + } + + @Override + public void moveToCurrentRow() throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("moveToCurrentRow not implemented"); + } + + @Override + public void moveToInsertRow() throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("moveToInsertRow not implemented"); + } + + @Override + public boolean previous() throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("previous not implemented"); + } + + @Override + public void refreshRow() throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("refreshRow not implemented"); + } + + @Override + public boolean relative(int rows) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("relative not implemented"); + } + + @Override + public boolean rowDeleted() throws SQLException { + ensureOpen(); + + return false; + } + + @Override + public boolean rowInserted() throws SQLException { + ensureOpen(); + + return false; + } + + @Override + public boolean rowUpdated() throws SQLException { + ensureOpen(); + + return false; + } + + @Override + public void setFetchDirection(int direction) throws SQLException { + ensureOpen(); + + if (direction != ResultSet.FETCH_FORWARD) { + throw SqlExceptionUtils.unsupportedError("only FETCH_FORWARD is supported in setFetchDirection"); + } + } + + @Override + public void updateArray(int columnIndex, Array x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateArray not implemented"); + } + + @Override + public void updateArray(String columnLabel, Array x) throws SQLException { + updateArray(findColumn(columnLabel), x); + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateAsciiStream not implemented"); + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException { + updateAsciiStream(findColumn(columnLabel), x); + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateAsciiStream not implemented"); + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { + updateAsciiStream(findColumn(columnLabel), x, length); + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateAsciiStream not implemented"); + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException { + updateAsciiStream(findColumn(columnLabel), x, length); + } + + @Override + public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateBigDecimal not implemented"); + } + + @Override + public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { + updateBigDecimal(findColumn(columnLabel), x); + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateBinaryStream not implemented"); + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException { + updateBinaryStream(findColumn(columnLabel), x); + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateBinaryStream not implemented"); + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { + updateBinaryStream(findColumn(columnLabel), x, length); + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateBinaryStream not implemented"); + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException { + updateBinaryStream(findColumn(columnLabel), x, length); + } + + @Override + public void updateBlob(int columnIndex, Blob x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateBlob not implemented"); + } + + @Override + public void updateBlob(String columnLabel, Blob x) throws SQLException { + updateBlob(findColumn(columnLabel), x); + } + + @Override + public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateBlob not implemented"); + } + + @Override + public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException { + updateBlob(findColumn(columnLabel), inputStream); + } + + @Override + public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateBlob not implemented"); + } + + @Override + public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException { + updateBlob(findColumn(columnLabel), inputStream, length); + } + + @Override + public void updateBoolean(int columnIndex, boolean x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateBoolean not implemented"); + } + + @Override + public void updateBoolean(String columnLabel, boolean x) throws SQLException { + updateBoolean(findColumn(columnLabel), x); + } + + @Override + public void updateByte(int columnIndex, byte x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateByte not implemented"); + } + + @Override + public void updateByte(String columnLabel, byte x) throws SQLException { + updateByte(findColumn(columnLabel), x); + } + + @Override + public void updateBytes(int columnIndex, byte[] x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateBytes not implemented"); + } + + @Override + public void updateBytes(String columnLabel, byte[] x) throws SQLException { + updateBytes(findColumn(columnLabel), x); + } + + @Override + public void updateCharacterStream(int columnIndex, Reader reader) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateCharacterStream not implemented"); + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException { + updateCharacterStream(findColumn(columnLabel), reader); + } + + @Override + public void updateCharacterStream(int columnIndex, Reader reader, int length) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateCharacterStream not implemented"); + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { + updateCharacterStream(findColumn(columnLabel), reader, length); + } + + @Override + public void updateCharacterStream(int columnIndex, Reader reader, long length) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateCharacterStream not implemented"); + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { + updateCharacterStream(findColumn(columnLabel), reader, length); + } + + @Override + public void updateClob(int columnIndex, Clob x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateClob not implemented"); + } + + @Override + public void updateClob(String columnLabel, Clob x) throws SQLException { + updateClob(findColumn(columnLabel), x); + } + + @Override + public void updateClob(int columnIndex, Reader reader) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateClob not implemented"); + } + + @Override + public void updateClob(String columnLabel, Reader reader) throws SQLException { + updateClob(findColumn(columnLabel), reader); + } + + @Override + public void updateClob(int columnIndex, Reader reader, long length) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateClob not implemented"); + } + + @Override + public void updateClob(String columnLabel, Reader reader, long length) throws SQLException { + updateClob(findColumn(columnLabel), reader, length); + } + + @Override + public void updateDate(int columnIndex, Date x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateDate not implemented"); + } + + @Override + public void updateDate(String columnLabel, Date x) throws SQLException { + updateDate(findColumn(columnLabel), x); + } + + @Override + public void updateDouble(int columnIndex, double x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateDouble not implemented"); + } + + @Override + public void updateDouble(String columnLabel, double x) throws SQLException { + updateDouble(findColumn(columnLabel), x); + } + + @Override + public void updateFloat(int columnIndex, float x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateFloat not implemented"); + } + + @Override + public void updateFloat(String columnLabel, float x) throws SQLException { + updateFloat(findColumn(columnLabel), x); + } + + @Override + public void updateInt(int columnIndex, int x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateInt not implemented"); + } + + @Override + public void updateInt(String columnLabel, int x) throws SQLException { + updateInt(findColumn(columnLabel), x); + } + + @Override + public void updateLong(int columnIndex, long x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateLong not implemented"); + } + + @Override + public void updateLong(String columnLabel, long x) throws SQLException { + updateLong(findColumn(columnLabel), x); + } + + @Override + public void updateNCharacterStream(int columnIndex, Reader reader) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateNCharacterStream not implemented"); + } + + @Override + public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException { + updateNCharacterStream(findColumn(columnLabel), reader); + } + + @Override + public void updateNCharacterStream(int columnIndex, Reader reader, long length) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateNCharacterStream not implemented"); + } + + @Override + public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { + updateNCharacterStream(findColumn(columnLabel), reader, length); + } + + @Override + public void updateNClob(int columnIndex, NClob nClob) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateNClob not implemented"); + } + + @Override + public void updateNClob(String columnLabel, NClob nClob) throws SQLException { + updateNClob(findColumn(columnLabel), nClob); + } + + @Override + public void updateNClob(int columnIndex, Reader reader) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateNClob not implemented"); + } + + @Override + public void updateNClob(String columnLabel, Reader reader) throws SQLException { + updateNClob(findColumn(columnLabel), reader); + } + + @Override + public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateNClob not implemented"); + } + + @Override + public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException { + updateNClob(findColumn(columnLabel), reader, length); + } + + @Override + public void updateNString(int columnIndex, String nString) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateNString not implemented"); + } + + @Override + public void updateNString(String columnLabel, String nString) throws SQLException { + updateNString(findColumn(columnLabel), nString); + } + + @Override + public void updateNull(int columnIndex) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateNull not implemented"); + } + + @Override + public void updateNull(String columnLabel) throws SQLException { + updateNull(findColumn(columnLabel)); + } + + @Override + public void updateObject(int columnIndex, Object x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateObject not implemented"); + } + + @Override + public void updateObject(String columnLabel, Object x) throws SQLException { + updateObject(findColumn(columnLabel), x); + } + + @Override + public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateObject not implemented"); + } + + @Override + public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { + updateObject(findColumn(columnLabel), x, scaleOrLength); + } + + @Override + public void updateObject(int columnIndex, Object x, SQLType targetSqlType) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateObject not implemented"); + } + + @Override + public void updateObject(String columnLabel, Object x, SQLType targetSqlType) throws SQLException { + updateObject(findColumn(columnLabel), x, targetSqlType); + } + + @Override + public void updateObject(int columnIndex, Object x, SQLType targetSqlType, int scaleOrLength) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateObject not implemented"); + } + + @Override + public void updateObject(String columnLabel, Object x, SQLType targetSqlType, int scaleOrLength) + throws SQLException { + updateObject(findColumn(columnLabel), x, targetSqlType, scaleOrLength); + } + + @Override + public void updateRef(int columnIndex, Ref x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateRef not implemented"); + } + + @Override + public void updateRef(String columnLabel, Ref x) throws SQLException { + updateRef(findColumn(columnLabel), x); + } + + @Override + public void updateRow() throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateRow not implemented"); + } + + @Override + public void updateRowId(int columnIndex, RowId x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateRowId not implemented"); + } + + @Override + public void updateRowId(String columnLabel, RowId x) throws SQLException { + updateRowId(findColumn(columnLabel), x); + } + + @Override + public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateSQLXML not implemented"); + } + + @Override + public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException { + updateSQLXML(findColumn(columnLabel), xmlObject); + } + + @Override + public void updateShort(int columnIndex, short x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateShort not implemented"); + } + + @Override + public void updateShort(String columnLabel, short x) throws SQLException { + updateShort(findColumn(columnLabel), x); + } + + @Override + public void updateString(int columnIndex, String x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateString not implemented"); + } + + @Override + public void updateString(String columnLabel, String x) throws SQLException { + updateString(findColumn(columnLabel), x); + } + + @Override + public void updateTime(int columnIndex, Time x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateTime not implemented"); + } + + @Override + public void updateTime(String columnLabel, Time x) throws SQLException { + updateTime(findColumn(columnLabel), x); + } + + @Override + public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("updateTimestamp not implemented"); + } + + @Override + public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { + updateTimestamp(findColumn(columnLabel), x); + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseArray.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseArray.java new file mode 100644 index 000000000..716165d33 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseArray.java @@ -0,0 +1,95 @@ +package com.clickhouse.jdbc; + +import java.sql.Array; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseColumn; + +public class ClickHouseArray implements Array { + private final int columnIndex; + private ClickHouseResultSet resultSet; + + protected ClickHouseArray(ClickHouseResultSet resultSet, int columnIndex) throws SQLException { + this.resultSet = ClickHouseChecker.nonNull(resultSet, "ResultSet"); + resultSet.ensureRead(columnIndex); + this.columnIndex = columnIndex; + } + + protected void ensureValid() throws SQLException { + if (resultSet == null) { + throw SqlExceptionUtils.clientError("Cannot operate on a freed Array object"); + } + } + + protected ClickHouseColumn getBaseColumn() { + return resultSet.columns.get(columnIndex - 1).getArrayBaseColumn(); + } + + @Override + public String getBaseTypeName() throws SQLException { + ensureValid(); + + return getBaseColumn().getDataType().name(); + } + + @Override + public int getBaseType() throws SQLException { + ensureValid(); + + return JdbcTypeMapping.toJdbcType(getBaseColumn()); + } + + @Override + public Object getArray() throws SQLException { + ensureValid(); + + return resultSet.getValue(columnIndex).asObject(); + } + + @Override + public Object getArray(Map> map) throws SQLException { + return getArray(); + } + + @Override + public Object getArray(long index, int count) throws SQLException { + ensureValid(); + + throw SqlExceptionUtils.unsupportedError("getArray not implemented"); + } + + @Override + public Object getArray(long index, int count, Map> map) throws SQLException { + return getArray(index, count); + } + + @Override + public ResultSet getResultSet() throws SQLException { + ensureValid(); + + throw SqlExceptionUtils.unsupportedError("getResultSet not implemented"); + } + + @Override + public ResultSet getResultSet(Map> map) throws SQLException { + return getResultSet(); + } + + @Override + public ResultSet getResultSet(long index, int count) throws SQLException { + return getResultSet(); + } + + @Override + public ResultSet getResultSet(long index, int count, Map> map) throws SQLException { + return getResultSet(); + } + + @Override + public void free() throws SQLException { + this.resultSet = null; + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseBlob.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseBlob.java new file mode 100644 index 000000000..2aa23e44d --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseBlob.java @@ -0,0 +1,76 @@ +package com.clickhouse.jdbc; + +import java.io.InputStream; +import java.io.OutputStream; +import java.sql.Blob; +import java.sql.SQLException; + +public class ClickHouseBlob implements Blob { + + @Override + public long length() throws SQLException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public byte[] getBytes(long pos, int length) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public InputStream getBinaryStream() throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public long position(byte[] pattern, long start) throws SQLException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public long position(Blob pattern, long start) throws SQLException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public int setBytes(long pos, byte[] bytes) throws SQLException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public int setBytes(long pos, byte[] bytes, int offset, int len) throws SQLException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public OutputStream setBinaryStream(long pos) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void truncate(long len) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void free() throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public InputStream getBinaryStream(long pos, long length) throws SQLException { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseClob.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseClob.java new file mode 100644 index 000000000..0a6b7afe4 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseClob.java @@ -0,0 +1,91 @@ +package com.clickhouse.jdbc; + +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Reader; +import java.io.Writer; +import java.sql.Clob; +import java.sql.NClob; +import java.sql.SQLException; + +public class ClickHouseClob implements NClob { + + @Override + public long length() throws SQLException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public String getSubString(long pos, int length) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Reader getCharacterStream() throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public InputStream getAsciiStream() throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public long position(String searchstr, long start) throws SQLException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public long position(Clob searchstr, long start) throws SQLException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public int setString(long pos, String str) throws SQLException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public int setString(long pos, String str, int offset, int len) throws SQLException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public OutputStream setAsciiStream(long pos) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Writer setCharacterStream(long pos) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void truncate(long len) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void free() throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public Reader getCharacterStream(long pos, long length) throws SQLException { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseConnection.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseConnection.java new file mode 100644 index 000000000..d31bd0cb6 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseConnection.java @@ -0,0 +1,108 @@ +package com.clickhouse.jdbc; + +import java.net.URI; +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.TimeZone; + +import com.clickhouse.client.ClickHouseVersion; + +public interface ClickHouseConnection extends Connection { + @Override + default ClickHouseStatement createStatement() throws SQLException { + return createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, + ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Override + default ClickHouseStatement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { + return createStatement(resultSetType, resultSetConcurrency, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Override + ClickHouseStatement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException; + + @Override + default CallableStatement prepareCall(String sql) throws SQLException { + return prepareCall(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, + ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Override + default CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + return prepareCall(sql, resultSetType, resultSetConcurrency, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Override + default CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, + int resultSetHoldability) throws SQLException { + throw SqlExceptionUtils.unsupportedError("prepareCall not implemented"); + } + + @Override + default PreparedStatement prepareStatement(String sql) throws SQLException { + return prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, + ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Override + default PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { + if (autoGeneratedKeys != Statement.NO_GENERATED_KEYS) { + // not entirely true, what if the table engine is JDBC? + throw SqlExceptionUtils.unsupportedError("Only NO_GENERATED_KEYS is supported"); + } + + return prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, + ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Override + default PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { + // not entirely true, what if the table engine is JDBC? + throw SqlExceptionUtils.unsupportedError("ClickHouse does not support auto generated keys"); + } + + @Override + default PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { + // not entirely true, what if the table engine is JDBC? + throw SqlExceptionUtils.unsupportedError("ClickHouse does not support auto generated keys"); + } + + @Override + default PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) + throws SQLException { + return prepareStatement(sql, resultSetType, resultSetConcurrency, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + /** + * Gets current database. {@link #getSchema()} is similar but it will check if + * connection is closed or not hence may throw {@link SQLException}. + * + * @return non-null database name + */ + String getCurrentDatabase(); + + /** + * Gets current user. + * + * @return non-null user name + */ + String getCurrentUser(); + + TimeZone getEffectiveTimeZone(); + + TimeZone getServerTimeZone(); + + ClickHouseVersion getServerVersion(); + + URI getUri(); + + boolean isJdbcCompliant(); + + String newQueryId(); +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDataSource.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDataSource.java new file mode 100644 index 000000000..2c4344cdd --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDataSource.java @@ -0,0 +1,114 @@ +package com.clickhouse.jdbc; + +import javax.sql.DataSource; + +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.config.ClickHouseDefaults; +import com.clickhouse.jdbc.internal.ClickHouseJdbcUrlParser; +import com.clickhouse.jdbc.internal.ClickHouseJdbcUrlParser.ConnectionInfo; + +import java.io.PrintWriter; +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.Properties; +import java.util.logging.Logger; + +public class ClickHouseDataSource extends Wrapper implements DataSource { + private final String url; + + protected final ClickHouseDriver driver = new ClickHouseDriver(); + + protected final Properties properties; + protected final ClickHouseNode server; + protected final URI uri; + + protected PrintWriter printWriter; + protected int loginTimeoutSeconds = 0; + + public ClickHouseDataSource(String url) { + this(url, new Properties()); + } + + public ClickHouseDataSource(String url, Properties properties) { + if (url == null) { + throw new IllegalArgumentException("Incorrect ClickHouse jdbc url. It must be not null"); + } + this.url = url; + + try { + ConnectionInfo connInfo = ClickHouseJdbcUrlParser.parse(url, properties); + this.properties = connInfo.getProperties(); + this.server = connInfo.getServer(); + this.uri = connInfo.getUri(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public ClickHouseConnection getConnection() throws SQLException { + return driver.connect(url, properties); + } + + @Override + public ClickHouseConnection getConnection(String username, String password) throws SQLException { + if (username == null || username.isEmpty()) { + throw SqlExceptionUtils.clientError("Non-empty user name is required"); + } + + if (password == null) { + password = ""; + } + + Properties props = new Properties(properties); + props.setProperty(ClickHouseDefaults.USER.getKey(), username); + props.setProperty(ClickHouseDefaults.PASSWORD.getKey(), password); + return driver.connect(url, props); + } + + public String getHost() { + return server.getHost(); + } + + public int getPort() { + return server.getPort(); + } + + public String getDatabase() { + return server.getDatabase().orElse((String) ClickHouseDefaults.DATABASE.getEffectiveDefaultValue()); + } + + // public String getUrl() { + // return url; + // } + + public Properties getProperties() { + return properties; + } + + @Override + public PrintWriter getLogWriter() throws SQLException { + return printWriter; + } + + @Override + public void setLogWriter(PrintWriter out) throws SQLException { + printWriter = out; + } + + @Override + public void setLoginTimeout(int seconds) throws SQLException { + loginTimeoutSeconds = seconds; + } + + @Override + public int getLoginTimeout() throws SQLException { + return loginTimeoutSeconds; + } + + public Logger getParentLogger() throws SQLFeatureNotSupportedException { + return ClickHouseDriver.parentLogger; + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaData.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaData.java new file mode 100644 index 000000000..6aee3f297 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaData.java @@ -0,0 +1,1264 @@ +package com.clickhouse.jdbc; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.RowIdLifetime; +import java.sql.SQLException; +import java.sql.Types; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.client.ClickHouseParameterizedQuery; +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.client.data.ClickHouseRecordTransformer; +import com.clickhouse.client.data.ClickHouseSimpleResponse; +import com.clickhouse.client.logging.Logger; +import com.clickhouse.client.logging.LoggerFactory; + +public class ClickHouseDatabaseMetaData extends Wrapper implements DatabaseMetaData { + private static final Logger log = LoggerFactory.getLogger(ClickHouseDatabaseMetaData.class); + + private static final String DATABASE_NAME = "ClickHouse"; + private static final String DRIVER_NAME = DATABASE_NAME + " JDBC Driver"; + + private static final String[] TABLE_TYPES = new String[] { "DICTIONARY", "LOG TABLE", "MEMORY TABLE", + "REMOTE TABLE", "TABLE", "VIEW", "SYSTEM TABLE", "TEMPORARY TABLE" }; + + private final ClickHouseConnection connection; + + protected ResultSet empty(String columns) throws SQLException { + return fixed(columns, null); + } + + protected ResultSet fixed(String columns, Object[][] values) throws SQLException { + return new ClickHouseResultSet("", "", connection.createStatement(), + ClickHouseSimpleResponse.of(ClickHouseColumn.parse(columns), values)); + } + + protected ResultSet query(String sql) throws SQLException { + return query(sql, null, false); + } + + protected ResultSet query(String sql, boolean ignoreError) throws SQLException { + return query(sql, null, ignoreError); + } + + protected ResultSet query(String sql, ClickHouseRecordTransformer func) throws SQLException { + return query(sql, func, false); + } + + protected ResultSet query(String sql, ClickHouseRecordTransformer func, boolean ignoreError) throws SQLException { + SQLException error = null; + try { + ClickHouseStatement stmt = connection.createStatement(); + return new ClickHouseResultSet("", "", stmt, + // load everything into memory + ClickHouseSimpleResponse.of(stmt.getRequest().query(sql).execute().get(), func)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw SqlExceptionUtils.forCancellation(e); + } catch (Exception e) { + error = SqlExceptionUtils.handle(e); + } + + if (ignoreError) { + return null; + } else { + throw error; + } + } + + public ClickHouseDatabaseMetaData(ClickHouseConnection connection) { + this.connection = ClickHouseChecker.nonNull(connection, "Connection"); + } + + @Override + public boolean allProceduresAreCallable() throws SQLException { + return true; + } + + @Override + public boolean allTablesAreSelectable() throws SQLException { + return true; + } + + @Override + public String getURL() throws SQLException { + return connection.getUri().toString(); + } + + @Override + public String getUserName() throws SQLException { + return connection.getCurrentUser(); + } + + @Override + public boolean isReadOnly() throws SQLException { + return connection.isReadOnly(); + } + + @Override + public boolean nullsAreSortedHigh() throws SQLException { + return true; + } + + @Override + public boolean nullsAreSortedLow() throws SQLException { + return false; + } + + @Override + public boolean nullsAreSortedAtStart() throws SQLException { + return false; + } + + @Override + public boolean nullsAreSortedAtEnd() throws SQLException { + return false; + } + + @Override + public String getDatabaseProductName() throws SQLException { + return DATABASE_NAME; + } + + @Override + public String getDatabaseProductVersion() throws SQLException { + return connection.getServerVersion().toString(); + } + + @Override + public String getDriverName() throws SQLException { + return DRIVER_NAME; + } + + @Override + public String getDriverVersion() throws SQLException { + return ClickHouseDriver.driverVersionString; + } + + @Override + public int getDriverMajorVersion() { + return ClickHouseDriver.driverVersion.getMajorVersion(); + } + + @Override + public int getDriverMinorVersion() { + return ClickHouseDriver.driverVersion.getMinorVersion(); + } + + @Override + public boolean usesLocalFiles() throws SQLException { + return false; + } + + @Override + public boolean usesLocalFilePerTable() throws SQLException { + return false; + } + + @Override + public boolean supportsMixedCaseIdentifiers() throws SQLException { + return true; + } + + @Override + public boolean storesUpperCaseIdentifiers() throws SQLException { + return false; + } + + @Override + public boolean storesLowerCaseIdentifiers() throws SQLException { + return false; + } + + @Override + public boolean storesMixedCaseIdentifiers() throws SQLException { + return true; + } + + @Override + public boolean supportsMixedCaseQuotedIdentifiers() throws SQLException { + return true; + } + + @Override + public boolean storesUpperCaseQuotedIdentifiers() throws SQLException { + return false; + } + + @Override + public boolean storesLowerCaseQuotedIdentifiers() throws SQLException { + return false; + } + + @Override + public boolean storesMixedCaseQuotedIdentifiers() throws SQLException { + return true; + } + + @Override + public String getIdentifierQuoteString() throws SQLException { + return "`"; + } + + @Override + public String getSQLKeywords() throws SQLException { + return "APPLY,ASOF,ATTACH,CLUSTER,DATABASE,DATABASES,DETACH," + + "DICTIONARY,DICTIONARIES,ILIKE,INF,LIMIT,LIVE,KILL,MATERIALIZED," + + "NAN,OFFSET,OPTIMIZE,OUTFILE,POLICY,PREWHERE,PROFILE,QUARTER,QUOTA," + + "RENAME,REPLACE,SAMPLE,SETTINGS,SHOW,TABLES,TIES,TOP,TOTALS,TRUNCATE,USE,WATCH,WEEK"; + } + + @Override + public String getNumericFunctions() throws SQLException { + // took from below URLs(not from system.functions): + // https://clickhouse.com/docs/en/sql-reference/functions/arithmetic-functions/ + // https://clickhouse.com/docs/en/sql-reference/functions/math-functions/ + return "abs,acos,acosh,asin,asinh,atan,atan2,atanh,cbrt,cos,cosh,divide,e,erf,erfc,exp,exp10,exp2,gcd,hypot,intDiv,intDivOrZero,intExp10,intExp2,lcm,lgamma,ln,log,log10,log1p,log2,minus,modulo,moduloOrZero,multiply,negate,pi,plus,pow,power,sign,sin,sinh,sqrt,tan,tgamma"; + } + + @Override + public String getStringFunctions() throws SQLException { + // took from below URLs(not from system.functions): + // https://clickhouse.com/docs/en/sql-reference/functions/string-functions/ + // https://clickhouse.com/docs/en/sql-reference/functions/string-search-functions/ + // https://clickhouse.com/docs/en/sql-reference/functions/string-replace-functions/ + return "appendTrailingCharIfAbsent,base64Decode,base64Encode,char_length,CHAR_LENGTH,character_length,CHARACTER_LENGTH,concat,concatAssumeInjective,convertCharset,countMatches,countSubstrings,countSubstringsCaseInsensitive,countSubstringsCaseInsensitiveUTF8,CRC32,CRC32IEEE,CRC64,decodeXMLComponent,empty,encodeXMLComponent,endsWith,extract,extractAll,extractAllGroupsHorizontal,extractAllGroupsVertical,extractTextFromHTML ,format,ilike,isValidUTF8,lcase,leftPad,leftPadUTF8,length,lengthUTF8,like,locate,lower,lowerUTF8,match,mid,multiFuzzyMatchAllIndices,multiFuzzyMatchAny,multiFuzzyMatchAnyIndex,multiMatchAllIndices,multiMatchAny,multiMatchAnyIndex,multiSearchAllPositions,multiSearchAllPositionsUTF8,multiSearchAny,multiSearchFirstIndex,multiSearchFirstPosition,ngramDistance,ngramSearch,normalizedQueryHash,normalizeQuery,notEmpty,notLike,position,positionCaseInsensitive,positionCaseInsensitiveUTF8,positionUTF8,regexpQuoteMeta,repeat,replace,replaceAll,replaceOne,replaceRegexpAll,replaceRegexpOne,reverse,reverseUTF8,rightPad,rightPadUTF8,startsWith,substr,substring,substringUTF8,tokens,toValidUTF8,trim,trimBoth,trimLeft,trimRight,tryBase64Decode,ucase,upper,upperUTF8"; + } + + @Override + public String getSystemFunctions() throws SQLException { + // took from below URL(not from system.functions): + // https://clickhouse.com/docs/en/sql-reference/functions/other-functions/ + return "bar,basename,blockNumber,blockSerializedSize,blockSize,buildId,byteSize,countDigits,currentDatabase,currentProfiles,currentRoles,currentUser,defaultProfiles,defaultRoles,defaultValueOfArgumentType,defaultValueOfTypeName,dumpColumnStructure,enabledProfiles,enabledRoles,errorCodeToName,filesystemAvailable,filesystemCapacity,filesystemFree,finalizeAggregation,formatReadableQuantity,formatReadableSize,formatReadableTimeDelta,FQDN,getMacro,getServerPort,getSetting,getSizeOfEnumType,greatest,hasColumnInTable,hostName,identity,ifNotFinite,ignore,indexHint,initializeAggregation,initialQueryID,isConstant,isDecimalOverflow,isFinite,isInfinite,isNaN,joinGet,least,MACNumToString,MACStringToNum,MACStringToOUI,materialize,modelEvaluate,neighbor,queryID,randomFixedString,randomPrintableASCII,randomString,randomStringUTF8,replicate,rowNumberInAllBlocks,rowNumberInBlock,runningAccumulate,runningConcurrency,runningDifference,runningDifferenceStartingWithFirstValue,shardCount ,shardNum,sleep,sleepEachRow,tcpPort,throwIf,toColumnTypeName,toTypeName,transform,uptime,version,visibleWidth"; + } + + @Override + public String getTimeDateFunctions() throws SQLException { + // took from below URL(not from system.functions): + // https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/ + return "addDays,addHours,addMinutes,addMonths,addQuarters,addSeconds,addWeeks,addYears,date_add,date_diff,date_sub,date_trunc,dateName,formatDateTime,FROM_UNIXTIME,fromModifiedJulianDay,fromModifiedJulianDayOrNull,now,subtractDays,subtractHours,subtractMinutes,subtractMonths,subtractQuarters,subtractSeconds,subtractWeeks,subtractYears,timeSlot,timeSlots,timestamp_add,timestamp_sub,timeZone,timeZoneOf,timeZoneOffset,today,toDayOfMonth,toDayOfWeek,toDayOfYear,toHour,toISOWeek,toISOYear,toMinute,toModifiedJulianDay,toModifiedJulianDayOrNull,toMonday,toMonth,toQuarter,toRelativeDayNum,toRelativeHourNum,toRelativeMinuteNum,toRelativeMonthNum,toRelativeQuarterNum,toRelativeSecondNum,toRelativeWeekNum,toRelativeYearNum,toSecond,toStartOfDay,toStartOfFifteenMinutes,toStartOfFiveMinute,toStartOfHour,toStartOfInterval,toStartOfISOYear,toStartOfMinute,toStartOfMonth,toStartOfQuarter,toStartOfSecond,toStartOfTenMinutes,toStartOfWeek,toStartOfYear,toTime,toTimeZone,toUnixTimestamp,toWeek,toYear,toYearWeek,toYYYYMM,toYYYYMMDD,toYYYYMMDDhhmmss,yesterday"; + } + + @Override + public String getSearchStringEscape() throws SQLException { + return "\\"; + } + + @Override + public String getExtraNameCharacters() throws SQLException { + return ""; + } + + @Override + public boolean supportsAlterTableWithAddColumn() throws SQLException { + return true; + } + + @Override + public boolean supportsAlterTableWithDropColumn() throws SQLException { + return true; + } + + @Override + public boolean supportsColumnAliasing() throws SQLException { + return true; + } + + @Override + public boolean nullPlusNonNullIsNull() throws SQLException { + return true; + } + + @Override + public boolean supportsConvert() throws SQLException { + // TODO select { fn CONVERT({ts '2021-01-01 12:12:12'}, TIMESTAMP) } + // select cast('2021-01-01 12:12:12' as DateTime) + return false; + } + + @Override + public boolean supportsConvert(int fromType, int toType) throws SQLException { + // TODO select { fn CONVERT({ts '2021-01-01 12:12:12'}, TIMESTAMP) } + // select cast('2021-01-01 12:12:12' as DateTime) + return false; + } + + @Override + public boolean supportsTableCorrelationNames() throws SQLException { + return true; + } + + @Override + public boolean supportsDifferentTableCorrelationNames() throws SQLException { + return false; + } + + @Override + public boolean supportsExpressionsInOrderBy() throws SQLException { + return true; + } + + @Override + public boolean supportsOrderByUnrelated() throws SQLException { + return true; + } + + @Override + public boolean supportsGroupBy() throws SQLException { + return true; + } + + @Override + public boolean supportsGroupByUnrelated() throws SQLException { + return true; + } + + @Override + public boolean supportsGroupByBeyondSelect() throws SQLException { + return true; + } + + @Override + public boolean supportsLikeEscapeClause() throws SQLException { + return true; + } + + @Override + public boolean supportsMultipleResultSets() throws SQLException { + // TODO let's add this in 0.3.3 + return false; + } + + @Override + public boolean supportsMultipleTransactions() throws SQLException { + return false; + } + + @Override + public boolean supportsNonNullableColumns() throws SQLException { + return true; + } + + @Override + public boolean supportsMinimumSQLGrammar() throws SQLException { + return true; + } + + @Override + public boolean supportsCoreSQLGrammar() throws SQLException { + return true; + } + + @Override + public boolean supportsExtendedSQLGrammar() throws SQLException { + return false; + } + + @Override + public boolean supportsANSI92EntryLevelSQL() throws SQLException { + return true; + } + + @Override + public boolean supportsANSI92IntermediateSQL() throws SQLException { + return false; + } + + @Override + public boolean supportsANSI92FullSQL() throws SQLException { + return false; + } + + @Override + public boolean supportsIntegrityEnhancementFacility() throws SQLException { + return false; + } + + @Override + public boolean supportsOuterJoins() throws SQLException { + return true; + } + + @Override + public boolean supportsFullOuterJoins() throws SQLException { + return true; + } + + @Override + public boolean supportsLimitedOuterJoins() throws SQLException { + return true; + } + + @Override + public String getSchemaTerm() throws SQLException { + return "database"; + } + + @Override + public String getProcedureTerm() throws SQLException { + return "procedure"; + } + + @Override + public String getCatalogTerm() throws SQLException { + return "catalog"; + } + + @Override + public boolean isCatalogAtStart() throws SQLException { + return false; + } + + @Override + public String getCatalogSeparator() throws SQLException { + return "."; + } + + @Override + public boolean supportsSchemasInDataManipulation() throws SQLException { + return true; + } + + @Override + public boolean supportsSchemasInProcedureCalls() throws SQLException { + return true; + } + + @Override + public boolean supportsSchemasInTableDefinitions() throws SQLException { + return true; + } + + @Override + public boolean supportsSchemasInIndexDefinitions() throws SQLException { + return true; + } + + @Override + public boolean supportsSchemasInPrivilegeDefinitions() throws SQLException { + return true; + } + + @Override + public boolean supportsCatalogsInDataManipulation() throws SQLException { + return false; + } + + @Override + public boolean supportsCatalogsInProcedureCalls() throws SQLException { + return false; + } + + @Override + public boolean supportsCatalogsInTableDefinitions() throws SQLException { + return false; + } + + @Override + public boolean supportsCatalogsInIndexDefinitions() throws SQLException { + return false; + } + + @Override + public boolean supportsCatalogsInPrivilegeDefinitions() throws SQLException { + return false; + } + + @Override + public boolean supportsPositionedDelete() throws SQLException { + return false; + } + + @Override + public boolean supportsPositionedUpdate() throws SQLException { + return false; + } + + @Override + public boolean supportsSelectForUpdate() throws SQLException { + return false; + } + + @Override + public boolean supportsStoredProcedures() throws SQLException { + return false; + } + + @Override + public boolean supportsSubqueriesInComparisons() throws SQLException { + return true; + } + + @Override + public boolean supportsSubqueriesInExists() throws SQLException { + return false; + } + + @Override + public boolean supportsSubqueriesInIns() throws SQLException { + return true; + } + + @Override + public boolean supportsSubqueriesInQuantifieds() throws SQLException { + return true; + } + + @Override + public boolean supportsCorrelatedSubqueries() throws SQLException { + return true; + } + + @Override + public boolean supportsUnion() throws SQLException { + return true; + } + + @Override + public boolean supportsUnionAll() throws SQLException { + return true; + } + + @Override + public boolean supportsOpenCursorsAcrossCommit() throws SQLException { + return false; + } + + @Override + public boolean supportsOpenCursorsAcrossRollback() throws SQLException { + return false; + } + + @Override + public boolean supportsOpenStatementsAcrossCommit() throws SQLException { + return false; + } + + @Override + public boolean supportsOpenStatementsAcrossRollback() throws SQLException { + return false; + } + + @Override + public int getMaxBinaryLiteralLength() throws SQLException { + return 0; + } + + @Override + public int getMaxCharLiteralLength() throws SQLException { + return 0; + } + + @Override + public int getMaxColumnNameLength() throws SQLException { + return 0; + } + + @Override + public int getMaxColumnsInGroupBy() throws SQLException { + return 0; + } + + @Override + public int getMaxColumnsInIndex() throws SQLException { + return 0; + } + + @Override + public int getMaxColumnsInOrderBy() throws SQLException { + return 0; + } + + @Override + public int getMaxColumnsInSelect() throws SQLException { + return 0; + } + + @Override + public int getMaxColumnsInTable() throws SQLException { + return 0; + } + + @Override + public int getMaxConnections() throws SQLException { + return 0; + } + + @Override + public int getMaxCursorNameLength() throws SQLException { + return 0; + } + + @Override + public int getMaxIndexLength() throws SQLException { + return 0; + } + + @Override + public int getMaxSchemaNameLength() throws SQLException { + return 0; + } + + @Override + public int getMaxProcedureNameLength() throws SQLException { + return 0; + } + + @Override + public int getMaxCatalogNameLength() throws SQLException { + return 0; + } + + @Override + public int getMaxRowSize() throws SQLException { + return 0; + } + + @Override + public boolean doesMaxRowSizeIncludeBlobs() throws SQLException { + return true; + } + + @Override + public int getMaxStatementLength() throws SQLException { + return 0; + } + + @Override + public int getMaxStatements() throws SQLException { + return 0; + } + + @Override + public int getMaxTableNameLength() throws SQLException { + return 0; + } + + @Override + public int getMaxTablesInSelect() throws SQLException { + return 0; + } + + @Override + public int getMaxUserNameLength() throws SQLException { + return 0; + } + + @Override + public int getDefaultTransactionIsolation() throws SQLException { + return connection.isJdbcCompliant() ? Connection.TRANSACTION_READ_COMMITTED : Connection.TRANSACTION_NONE; + } + + @Override + public boolean supportsTransactions() throws SQLException { + return connection.isJdbcCompliant(); + } + + @Override + public boolean supportsTransactionIsolationLevel(int level) throws SQLException { + if (Connection.TRANSACTION_NONE == level) { + return true; + } else if (Connection.TRANSACTION_READ_UNCOMMITTED != level && Connection.TRANSACTION_READ_COMMITTED != level + && Connection.TRANSACTION_REPEATABLE_READ != level && Connection.TRANSACTION_SERIALIZABLE != level) { + throw SqlExceptionUtils.clientError("Unknown isolation level: " + level); + } + + return connection.isJdbcCompliant(); + } + + @Override + public boolean supportsDataDefinitionAndDataManipulationTransactions() throws SQLException { + return false; + } + + @Override + public boolean supportsDataManipulationTransactionsOnly() throws SQLException { + return connection.isJdbcCompliant(); + } + + @Override + public boolean dataDefinitionCausesTransactionCommit() throws SQLException { + return false; + } + + @Override + public boolean dataDefinitionIgnoredInTransactions() throws SQLException { + return false; + } + + @Override + public ResultSet getProcedures(String catalog, String schemaPattern, String procedureNamePattern) + throws SQLException { + return empty("PROCEDURE_CAT Nullable(String), PROCEDURE_SCHEM Nullable(String), " + + "RESERVED1 Nullable(String), RESERVED2 Nullable(String), RESERVED3 Nullable(String), " + + "PROCEDURE_NAME String, REMARKS String, PROCEDURE_TYPE Int16, SPECIFIC_NAME String"); + } + + @Override + public ResultSet getProcedureColumns(String catalog, String schemaPattern, String procedureNamePattern, + String columnNamePattern) throws SQLException { + return empty("PROCEDURE_CAT Nullable(String), PROCEDURE_SCHEM Nullable(String), " + + "PROCEDURE_NAME String, COLUMN_NAME String, COLUMN_TYPE Int16, " + + "DATA_TYPE Int32, TYPE_NAME String, PRECISION Int32, LENGTH Int32, " + + "SCALE Int16, RADIX Int16, NULLABLE Int16, REMARKS String, " + + "COLUMN_DEF Nullable(String), SQL_DATA_TYPE Int32, SQL_DATETIME_SUB Int32, " + + "CHAR_OCTET_LENGTH Int32, ORDINAL_POSITION Int32, IS_NULLABLE String, SPECIFIC_NAME String"); + } + + @Override + public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) + throws SQLException { + StringBuilder builder = new StringBuilder(); + if (types == null || types.length == 0) { + types = TABLE_TYPES; + } + for (String type : types) { + builder.append('\'').append(ClickHouseUtils.escape(type, '\'')).append('\'').append(','); + } + builder.setLength(builder.length() - 1); + + List databases = new LinkedList<>(); + if (ClickHouseChecker.isNullOrEmpty(schemaPattern)) { + try (ResultSet rs = query("select name from system.databases order by name")) { + while (rs.next()) { + databases.add(rs.getString(1)); + } + } catch (Exception e) { + // ignore + } finally { + if (databases.isEmpty()) { + databases.add("%"); + } + } + } else { + databases.add(schemaPattern); + } + + List results = new ArrayList<>(databases.size()); + for (String database : databases) { + Map params = new HashMap<>(); + params.put("comment", connection.getServerVersion().check("[18.16,)") ? "t.comment" : "''"); + params.put("database", ClickHouseValues.convertToQuotedString(database)); + params.put("table", ClickHouseChecker.isNullOrEmpty(tableNamePattern) ? "'%'" + : ClickHouseValues.convertToQuotedString(tableNamePattern)); + params.put("types", builder.toString()); + String sql = JdbcParameterizedQuery + .apply("select null as TABLE_CAT, t.database as TABLE_SCHEM, t.name as TABLE_NAME, " + + "case when t.engine like '%Log' then 'LOG TABLE' " + + "when t.engine in ('Buffer', 'Memory', 'Set') then 'MEMORY TABLE' " + + "when t.is_temporary != 0 then 'TEMPORARY TABLE' " + + "when t.engine like '%View' then 'VIEW' when t.engine = 'Dictionary' then 'DICTIONARY' " + + "when t.engine like 'System%' then 'SYSTEM TABLE' " + + "when empty(t.data_paths) then 'REMOTE TABLE' else 'TABLE' end as TABLE_TYPE, " + + ":comment as REMARKS, null as TYPE_CAT, d.engine as TYPE_SCHEM, " + + "t.engine as TYPE_NAME, null as SELF_REFERENCING_COL_NAME, null as REF_GENERATION\n" + + "from system.tables t inner join system.databases d on t.database = d.name\n" + + "where t.database like :database and t.name like :table and TABLE_TYPE in (:types) " + + "order by t.database, t.name", params); + results.add(query(sql, true)); + } + return new CombinedResultSet(results); + } + + @Override + public ResultSet getSchemas() throws SQLException { + return getSchemas(null, null); + } + + @Override + public ResultSet getCatalogs() throws SQLException { + return empty("TABLE_CAT String"); + } + + @Override + public ResultSet getTableTypes() throws SQLException { + // "TABLE", "VIEW", "SYSTEM TABLE", "GLOBAL TEMPORARY", "LOCAL TEMPORARY", + // "ALIAS", "SYNONYM". + int len = TABLE_TYPES.length; + Object[][] rows = new Object[len][]; + for (int i = 0; i < len; i++) { + rows[i] = new Object[] { TABLE_TYPES[i] }; + } + return fixed("TABLE_TYPE String", rows); + } + + @Override + public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) + throws SQLException { + Map params = new HashMap<>(); + params.put("comment", connection.getServerVersion().check("[18.16,)") ? "comment" : "''"); + params.put("database", ClickHouseChecker.isNullOrEmpty(schemaPattern) ? "'%'" + : ClickHouseValues.convertToQuotedString(schemaPattern)); + params.put("table", ClickHouseChecker.isNullOrEmpty(tableNamePattern) ? "'%'" + : ClickHouseValues.convertToQuotedString(tableNamePattern)); + params.put("column", ClickHouseChecker.isNullOrEmpty(columnNamePattern) ? "'%'" + : ClickHouseValues.convertToQuotedString(columnNamePattern)); + params.put("defaultNullable", String.valueOf(DatabaseMetaData.typeNullable)); + params.put("defaultNonNull", String.valueOf(DatabaseMetaData.typeNoNulls)); + params.put("defaultType", String.valueOf(Types.OTHER)); + String sql = JdbcParameterizedQuery + .apply("select null as TABLE_CAT, database as TABLE_SCHEM, table as TABLE_NAME, " + + "name as COLUMN_NAME, toInt32(:defaultType) as DATA_TYPE, type as TYPE_NAME, toInt32(0) as COLUMN_SIZE, " + + "0 as BUFFER_LENGTH, toInt32(null) as DECIMAL_DIGITS, 10 as NUM_PREC_RADIX, " + + "toInt32(position(type, 'Nullable(') >= 1 ? :defaultNullable : :defaultNonNull) as NULLABLE, :comment as REMARKS, default_expression as COLUMN_DEF, " + + "0 as SQL_DATA_TYPE, 0 as SQL_DATETIME_SUB, toInt32(null) as CHAR_OCTET_LENGTH, position as ORDINAL_POSITION, " + + "position(type, 'Nullable(') >= 1 ? 'YES' : 'NO' as IS_NULLABLE, null as SCOPE_CATALOG, null as SCOPE_SCHEMA, null as SCOPE_TABLE, " + + "null as SOURCE_DATA_TYPE, 'NO' as IS_AUTOINCREMENT, 'NO' as IS_GENERATEDCOLUMN from system.columns\n" + + "where database like :database and table like :table and name like :column", params); + return query(sql, (i, r) -> { + String typeName = r.getValue("TYPE_NAME").asString(); + try { + ClickHouseColumn column = ClickHouseColumn.of("", typeName); + r.getValue("DATA_TYPE").update(JdbcTypeMapping.toJdbcType(column)); + r.getValue("COLUMN_SIZE").update(column.getDataType().getByteLength()); + if (column.isNullable()) { + r.getValue("NULLABLE").update(DatabaseMetaData.typeNullable); + r.getValue("IS_NULLABLE").update("YES"); + } else { + r.getValue("NULLABLE").update(DatabaseMetaData.typeNoNulls); + r.getValue("IS_NULLABLE").update("NO"); + } + + if (column.getDataType() == ClickHouseDataType.FixedString) { + r.getValue("CHAR_OCTET_LENGTH").update(column.getPrecision()); + } + + if (column.getScale() > 0) { + r.getValue("DECIMAL_DIGITS").update(column.getScale()); + } else { + r.getValue("DECIMAL_DIGITS").resetToNullOrEmpty(); + } + } catch (Exception e) { + log.warn("Failed to read column: %s", typeName, e); + } + }); + } + + @Override + public ResultSet getColumnPrivileges(String catalog, String schema, String table, String columnNamePattern) + throws SQLException { + return empty("TABLE_CAT Nullable(String), TABLE_SCHEM Nullable(String), TABLE_NAME String, " + + "COLUMN_NAME String, GRANTOR Nullable(String), GRANTEE String, PRIVILEGE String, " + + "IS_GRANTABLE Nullable(String)"); + } + + @Override + public ResultSet getTablePrivileges(String catalog, String schemaPattern, String tableNamePattern) + throws SQLException { + return empty("TABLE_CAT Nullable(String), TABLE_SCHEM Nullable(String), TABLE_NAME String, " + + "GRANTOR Nullable(String), GRANTEE String, PRIVILEGE String, IS_GRANTABLE Nullable(String)"); + } + + @Override + public ResultSet getBestRowIdentifier(String catalog, String schema, String table, int scope, boolean nullable) + throws SQLException { + return getVersionColumns(catalog, schema, table); + } + + @Override + public ResultSet getVersionColumns(String catalog, String schema, String table) throws SQLException { + return empty("SCOPE Int16, COLUMN_NAME String, DATA_TYPE Int32, TYPE_NAME String, " + + "COLUMN_SIZE Int32, BUFFER_LENGTH Int32, DECIMAL_DIGITS Int16, PSEUDO_COLUMN Int16"); + } + + @Override + public ResultSet getPrimaryKeys(String catalog, String schema, String table) throws SQLException { + return empty("TABLE_CAT Nullable(String), TABLE_SCHEM Nullable(String), TABLE_NAME String, " + + "COLUMN_NAME String, KEY_SEQ Int16, PK_NAME String"); + } + + @Override + public ResultSet getImportedKeys(String catalog, String schema, String table) throws SQLException { + return empty("PKTABLE_CAT Nullable(String), PKTABLE_SCHEM Nullable(String), PKTABLE_NAME String, " + + "PKCOLUMN_NAME String, FKTABLE_CAT Nullable(String), FKTABLE_SCHEM Nullable(String), " + + "FKTABLE_NAME String, FKCOLUMN_NAME String, KEY_SEQ Int16, UPDATE_RULE Int16, " + + "DELETE_RULE Int16, FK_NAME Nullable(String), PK_NAME Nullable(String), DEFERRABILITY Int16"); + } + + @Override + public ResultSet getExportedKeys(String catalog, String schema, String table) throws SQLException { + return getImportedKeys(catalog, schema, table); + } + + @Override + public ResultSet getCrossReference(String parentCatalog, String parentSchema, String parentTable, + String foreignCatalog, String foreignSchema, String foreignTable) throws SQLException { + return empty("PKTABLE_CAT Nullable(String), PKTABLE_SCHEM Nullable(String), PKTABLE_NAME String, " + + "PKCOLUMN_NAME String, FKTABLE_CAT Nullable(String), FKTABLE_SCHEM Nullable(String), " + + "FKTABLE_NAME String, FKCOLUMN_NAME String, KEY_SEQ Int16, UPDATE_RULE Int16, " + + "DELETE_RULE Int16, FK_NAME Nullable(String), PK_NAME Nullable(String), DEFERRABILITY Int16"); + } + + private Object[] toTypeRow(String typeName, String aliasTo) { + ClickHouseDataType type; + try { + type = ClickHouseDataType.of(typeName); + } catch (Exception e) { + if (aliasTo == null || aliasTo.isEmpty()) { + return new Object[0]; + } + try { + type = ClickHouseDataType.of(aliasTo); + } catch (Exception ex) { + return new Object[0]; + } + } + + String prefix = ""; + String suffix = ""; + String params = ""; + int nullable = DatabaseMetaData.typeNullable; + int searchable = type == ClickHouseDataType.FixedString || type == ClickHouseDataType.String + ? DatabaseMetaData.typeSearchable + : DatabaseMetaData.typePredBasic; + int money = 0; + switch (type) { + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Enum: + case Enum8: + case Enum16: + case String: + case FixedString: + case UUID: + prefix = "'"; + suffix = "'"; + break; + case Array: + case Nested: + case Ring: + case Polygon: + case MultiPolygon: + prefix = "["; + suffix = "]"; + nullable = DatabaseMetaData.typeNoNulls; + break; + case AggregateFunction: + case Tuple: + case Point: + prefix = "("; + suffix = ")"; + nullable = DatabaseMetaData.typeNoNulls; + break; + case Map: + prefix = "{"; + suffix = "}"; + nullable = DatabaseMetaData.typeNoNulls; + break; + default: + break; + } + return new Object[] { typeName, + JdbcTypeMapping.toJdbcType(ClickHouseColumn.of("", type, false, false, new String[0])), + type.getMaxPrecision(), prefix, suffix, params, nullable, type.isCaseSensitive() ? 1 : 0, searchable, + type.getMaxPrecision() > 0 && !type.isSigned() ? 1 : 0, money, 0, + aliasTo == null || aliasTo.isEmpty() ? type.name() : aliasTo, type.getMinScale(), type.getMaxScale(), 0, + 0, 10 }; + } + + @Override + public ResultSet getTypeInfo() throws SQLException { + List list = new ArrayList<>(); + try (ResultSet rs = query("select name, alias_to from system.data_type_families order by name")) { + while (rs.next()) { + Object[] row = toTypeRow(rs.getString(1), rs.getString(2)); + if (row.length > 0) { + list.add(row); + } + } + } + + return fixed("TYPE_NAME String, DATA_TYPE Int32, PRECISION Int32, " + + "LITERAL_PREFIX Nullable(String), LITERAL_SUFFIX Nullable(String), CREATE_PARAMS Nullable(String), " + + "NULLABLE Int16, CASE_SENSITIVE UInt8, SEARCHABLE Int16, UNSIGNED_ATTRIBUTE UInt8, " + + "FIXED_PREC_SCALE UInt8, AUTO_INCREMENT UInt8, LOCAL_TYPE_NAME Nullable(String), " + + "MINIMUM_SCALE Int16, MAXIMUM_SCALE Int16, SQL_DATA_TYPE Int32, SQL_DATETIME_SUB Int32, " + + "NUM_PREC_RADIX Int32", list.toArray(new Object[0][])); + } + + @Override + public ResultSet getIndexInfo(String catalog, String schema, String table, boolean unique, boolean approximate) + throws SQLException { + Map params = new HashMap<>(); + params.put("database", + ClickHouseChecker.isNullOrEmpty(schema) ? "'%'" : ClickHouseValues.convertToQuotedString(schema)); + params.put("table", + ClickHouseChecker.isNullOrEmpty(table) ? "'%'" : ClickHouseValues.convertToQuotedString(table)); + params.put("statIndex", String.valueOf(DatabaseMetaData.tableIndexStatistic)); + params.put("otherIndex", String.valueOf(DatabaseMetaData.tableIndexOther)); + return new CombinedResultSet( + empty("TABLE_CAT Nullable(String), TABLE_SCHEM Nullable(String), TABLE_NAME String, " + + "NON_UNIQUE UInt8, INDEX_QUALIFIER Nullable(String), INDEX_NAME Nullable(String), " + + "TYPE Int16, ORDINAL_POSITION Int16, COLUMN_NAME Nullable(String), ASC_OR_DESC Nullable(String), " + + "CARDINALITY Int64, PAGES Int64, FILTER_CONDITION Nullable(String)"), + query(JdbcParameterizedQuery.apply( + "select null as TABLE_CAT, database as TABLE_SCHEM, table as TABLE_NAME, toUInt8(0) as NON_UNIQUE, " + + "null as INDEX_QUALIFIER, null as INDEX_NAME, toInt16(:statIndex) as TYPE, " + + "toInt16(0) as ORDINAL_POSITION, null as COLUMN_NAME, null as ASC_OR_DESC, " + + "sum(rows) as CARDINALITY, uniqExact(name) as PAGES, null as FILTER_CONDITION from system.parts " + + "where active = 1 and database like :database and table like :table group by database, table", + params), true), + query(JdbcParameterizedQuery.apply( + "select null as TABLE_CAT, database as TABLE_SCHEM, table as TABLE_NAME, toUInt8(1) as NON_UNIQUE, " + + "type as INDEX_QUALIFIER, name as INDEX_NAME, toInt16(:otherIndex) as TYPE, " + + "toInt16(1) as ORDINAL_POSITION, expr as COLUMN_NAME, null as ASC_OR_DESC, " + + "0 as CARDINALITY, 0 as PAGES, null as FILTER_CONDITION " + + "from system.data_skipping_indices where database like :database and table like :table", + params), true), + query(JdbcParameterizedQuery.apply( + "select null as TABLE_CAT, database as TABLE_SCHEM, table as TABLE_NAME, toUInt8(1) as NON_UNIQUE, " + + "null as INDEX_QUALIFIER, name as INDEX_NAME, toInt16(:otherIndex) as TYPE, " + + "column_position as ORDINAL_POSITION, column as COLUMN_NAME, null as ASC_OR_DESC, " + + "sum(rows) as CARDINALITY, uniqExact(partition) as PAGES, null as FILTER_CONDITION " + + "from system.projection_parts_columns where active = 1 and database like :database and table like :table " + + "group by database, table, name, column, column_position " + + "order by database, table, name, column_position", + params), true)); + } + + @Override + public boolean supportsResultSetType(int type) throws SQLException { + return ResultSet.TYPE_FORWARD_ONLY == type; + } + + @Override + public boolean supportsResultSetConcurrency(int type, int concurrency) throws SQLException { + return false; + } + + @Override + public boolean ownUpdatesAreVisible(int type) throws SQLException { + return true; + } + + @Override + public boolean ownDeletesAreVisible(int type) throws SQLException { + return true; + } + + @Override + public boolean ownInsertsAreVisible(int type) throws SQLException { + return true; + } + + @Override + public boolean othersUpdatesAreVisible(int type) throws SQLException { + return true; + } + + @Override + public boolean othersDeletesAreVisible(int type) throws SQLException { + return true; + } + + @Override + public boolean othersInsertsAreVisible(int type) throws SQLException { + return true; + } + + @Override + public boolean updatesAreDetected(int type) throws SQLException { + return false; + } + + @Override + public boolean deletesAreDetected(int type) throws SQLException { + return false; + } + + @Override + public boolean insertsAreDetected(int type) throws SQLException { + return false; + } + + @Override + public boolean supportsBatchUpdates() throws SQLException { + return true; + } + + @Override + public ResultSet getUDTs(String catalog, String schemaPattern, String typeNamePattern, int[] types) + throws SQLException { + return empty("TYPE_CAT Nullable(String), TYPE_SCHEM Nullable(String), TYPE_NAME String, " + + "CLASS_NAME String, DATA_TYPE Int32, REMARKS String, BASE_TYPE Int16"); + } + + @Override + public Connection getConnection() throws SQLException { + return connection; + } + + @Override + public boolean supportsSavepoints() throws SQLException { + return false; + } + + @Override + public boolean supportsNamedParameters() throws SQLException { + return false; + } + + @Override + public boolean supportsMultipleOpenResults() throws SQLException { + return false; + } + + @Override + public boolean supportsGetGeneratedKeys() throws SQLException { + return false; + } + + @Override + public ResultSet getSuperTypes(String catalog, String schemaPattern, String typeNamePattern) throws SQLException { + return empty("TYPE_CAT Nullable(String), TYPE_SCHEM Nullable(String), TYPE_NAME String, " + + "SUPERTYPE_CAT Nullable(String), SUPERTYPE_SCHEM Nullable(String), SUPERTYPE_NAME String"); + } + + @Override + public ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern) throws SQLException { + return empty( + "TABLE_CAT Nullable(String), TABLE_SCHEM Nullable(String), TABLE_NAME String, SUPERTABLE_NAME String"); + } + + @Override + public ResultSet getAttributes(String catalog, String schemaPattern, String typeNamePattern, + String attributeNamePattern) throws SQLException { + return empty("TYPE_CAT Nullable(String), TYPE_SCHEM Nullable(String), TYPE_NAME String, " + + "ATTR_NAME String, DATA_TYPE Int32, ATTR_TYPE_NAME String, ATTR_SIZE Int32, " + + "DECIMAL_DIGITS Int32, NUM_PREC_RADIX Int32, NULLABLE Int32, REMARKS Nullable(String), " + + "ATTR_DEF Nullable(String), SQL_DATA_TYPE Int32, SQL_DATETIME_SUB Int32, " + + "CHAR_OCTET_LENGTH Int32, ORDINAL_POSITION Int32, IS_NULLABLE String, " + + "SCOPE_CATALOG String, SCOPE_SCHEMA String, SCOPE_TABLE String, SOURCE_DATA_TYPE Int16"); + } + + @Override + public boolean supportsResultSetHoldability(int holdability) throws SQLException { + return false; + } + + @Override + public int getResultSetHoldability() throws SQLException { + return ResultSet.HOLD_CURSORS_OVER_COMMIT; + } + + @Override + public int getDatabaseMajorVersion() throws SQLException { + return connection.getServerVersion().getMajorVersion(); + } + + @Override + public int getDatabaseMinorVersion() throws SQLException { + return connection.getServerVersion().getMinorVersion(); + } + + @Override + public int getJDBCMajorVersion() throws SQLException { + return ClickHouseDriver.specVersion.getMajorVersion(); + } + + @Override + public int getJDBCMinorVersion() throws SQLException { + return ClickHouseDriver.specVersion.getMinorVersion(); + } + + @Override + public int getSQLStateType() throws SQLException { + return sqlStateSQL; + } + + @Override + public boolean locatorsUpdateCopy() throws SQLException { + return false; + } + + @Override + public boolean supportsStatementPooling() throws SQLException { + return false; + } + + @Override + public RowIdLifetime getRowIdLifetime() throws SQLException { + return RowIdLifetime.ROWID_UNSUPPORTED; + } + + @Override + public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLException { + Map params = Collections.singletonMap("pattern", + ClickHouseChecker.isNullOrEmpty(schemaPattern) ? "'%'" + : ClickHouseValues.convertToQuotedString(schemaPattern)); + return new CombinedResultSet( + query(JdbcParameterizedQuery.apply("select name as TABLE_SCHEM, null as TABLE_CATALOG " + + "from system.databases where name like :pattern order by name", params)), + query(JdbcParameterizedQuery.apply( + "select concat('jdbc(''', name, ''')') as TABLE_SCHEM, null as TABLE_CATALOG " + + "from jdbc('', 'SHOW DATASOURCES') where TABLE_SCHEM like :pattern order by name", + params), true)); + } + + @Override + public boolean supportsStoredFunctionsUsingCallSyntax() throws SQLException { + return false; + } + + @Override + public boolean autoCommitFailureClosesAllResultSets() throws SQLException { + return false; + } + + @Override + public ResultSet getClientInfoProperties() throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public ResultSet getFunctions(String catalog, String schemaPattern, String functionNamePattern) + throws SQLException { + String sql = JdbcParameterizedQuery.apply( + "select null as FUNCTION_CAT, null as FUNCTION_SCHEM, name as FUNCTION_NAME,\n" + + "concat('case-', case_insensitive ? 'in' : '', 'sensitive function', is_aggregate ? ' for aggregation' : '') as REMARKS," + + "1 as FUNCTION_TYPE, name as SPECIFIC_NAME from system.functions\n" + + "where alias_to = '' and name like :pattern order by name union all\n" + + "select null as FUNCTION_CAT, null as FUNCTION_SCHEM, name as FUNCTION_NAME,\n" + + "'case-sensistive table function' as REMARKS, 2 as FUNCTION_TYPE, name as SPECIFIC_NAME from system.table_functions\n" + + "order by name", + Collections.singletonMap("pattern", ClickHouseChecker.isNullOrEmpty(functionNamePattern) ? "'%'" + : ClickHouseValues.convertToQuotedString(functionNamePattern))); + + return query(sql); + } + + @Override + public ResultSet getFunctionColumns(String catalog, String schemaPattern, String functionNamePattern, + String columnNamePattern) throws SQLException { + return null; + } + + @Override + public ResultSet getPseudoColumns(String catalog, String schemaPattern, String tableNamePattern, + String columnNamePattern) throws SQLException { + return empty("TABLE_CAT Nullable(String), TABLE_SCHEM Nullable(String), TABLE_NAME String, " + + "COLUMN_NAME String, DATA_TYPE Int32, COLUMN_SIZE Int32, DECIMAL_DIGITS Int32, " + + "NUM_PREC_RADIX Int32, COLUMN_USAGE String, REMARKS Nullable(String), " + + "CHAR_OCTET_LENGTH Int32, IS_NULLABLE String"); + } + + @Override + public boolean generatedKeyAlwaysReturned() throws SQLException { + return false; + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDriver.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDriver.java new file mode 100644 index 000000000..2e3bdc3b5 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDriver.java @@ -0,0 +1,153 @@ +package com.clickhouse.jdbc; + +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.DriverPropertyInfo; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.ServiceLoader; +import java.util.WeakHashMap; + +import com.clickhouse.client.ClickHouseClient; +import com.clickhouse.client.ClickHouseVersion; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.client.logging.Logger; +import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.jdbc.internal.ClickHouseConnectionImpl; +import com.clickhouse.jdbc.internal.ClickHouseJdbcUrlParser; + +/** + * JDBC driver for ClickHouse. It takes a connection string like below for + * connecting to ClickHouse server: + * {@code jdbc:clickhouse://[:@][:][/][?parameter1=value1¶meter2=value2]} + * + *

+ * For examples: + *

    + *
  • {@code jdbc:clickhouse://localhost:8123/system}
    • + *
  • {@code jdbc:clickhouse://admin:password@localhost/system?socket_time=30}
    • + *
  • {@code jdbc:clickhouse://localhost/system?protocol=grpc}
    • + *
+ */ +public class ClickHouseDriver implements Driver { + private static final Logger log = LoggerFactory.getLogger(ClickHouseDriver.class); + + static final String driverVersionString; + static final ClickHouseVersion driverVersion; + static final ClickHouseVersion specVersion; + + static final java.util.logging.Logger parentLogger = java.util.logging.Logger.getLogger("com.clickhouse.jdbc"); + + static { + driverVersionString = ClickHouseDriver.class.getPackage().getImplementationVersion(); + driverVersion = ClickHouseVersion.of(driverVersionString); + specVersion = ClickHouseVersion.of(ClickHouseDriver.class.getPackage().getSpecificationVersion()); + + try { + DriverManager.registerDriver(new ClickHouseDriver()); + } catch (SQLException e) { + throw new IllegalStateException(e); + } + + log.debug("ClickHouse Driver %s(JDBC: %s) registered", driverVersion, specVersion); + } + + private DriverPropertyInfo create(ClickHouseOption option, Properties props) { + DriverPropertyInfo propInfo = new DriverPropertyInfo(option.getKey(), + props.getProperty(option.getKey(), String.valueOf(option.getEffectiveDefaultValue()))); + propInfo.required = false; + propInfo.description = option.getDescription(); + propInfo.choices = null; + + Class clazz = option.getValueType(); + if (Boolean.class == clazz || boolean.class == clazz) { + propInfo.choices = new String[] { "true", "false" }; + } else if (clazz.isEnum()) { + Object[] values = clazz.getEnumConstants(); + String[] names = new String[values.length]; + int index = 0; + for (Object v : values) { + names[index++] = ((Enum) v).name(); + } + propInfo.choices = names; + } + return propInfo; + } + + @Override + public boolean acceptsURL(String url) throws SQLException { + return url != null && (url.startsWith(ClickHouseJdbcUrlParser.JDBC_CLICKHOUSE_PREFIX) + || url.startsWith(ClickHouseJdbcUrlParser.JDBC_ABBREVIATION_PREFIX)); + } + + @Override + public ClickHouseConnection connect(String url, Properties info) throws SQLException { + if (!acceptsURL(url)) { + return null; + } + + log.debug("Creating connection"); + return new ClickHouseConnectionImpl(url, info); + } + + @Override + public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException { + try { + info = ClickHouseJdbcUrlParser.parse(url, info).getProperties(); + } catch (Exception e) { + log.error("Could not parse url %s", url, e); + } + + List result = new ArrayList<>(ClickHouseClientOption.values().length * 2); + for (ClickHouseClientOption option : ClickHouseClientOption.values()) { + result.add(create(option, info)); + } + + // client-specific configuration + try { + for (ClickHouseClient c : ServiceLoader.load(ClickHouseClient.class, getClass().getClassLoader())) { + Class clazz = c.getOptionClass(); + if (clazz == null || clazz == ClickHouseClientOption.class) { + continue; + } + for (ClickHouseOption option : clazz.getEnumConstants()) { + result.add(create(option, info)); + } + } + } catch (Exception e) { + log.warn("Failed to load client-specific configuration", e); + } + + DriverPropertyInfo custom = new DriverPropertyInfo(ClickHouseJdbcUrlParser.PROP_JDBC_COMPLIANT, "true"); + custom.choices = new String[] { "true", "false" }; + custom.description = "Whether to enable JDBC-compliant features like fake transaction and standard UPDATE and DELETE statements."; + result.add(custom); + return result.toArray(new DriverPropertyInfo[0]); + } + + @Override + public int getMajorVersion() { + return driverVersion.getMajorVersion(); + } + + @Override + public int getMinorVersion() { + return driverVersion.getMinorVersion(); + } + + @Override + public boolean jdbcCompliant() { + return false; + } + + @Override + public java.util.logging.Logger getParentLogger() throws SQLFeatureNotSupportedException { + return parentLogger; + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSet.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSet.java new file mode 100644 index 000000000..f51c8eff9 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSet.java @@ -0,0 +1,726 @@ +package com.clickhouse.jdbc; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.MalformedURLException; +import java.net.URL; +import java.sql.Array; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.Date; +import java.sql.NClob; +import java.sql.Ref; +import java.sql.ResultSetMetaData; +import java.sql.RowId; +import java.sql.SQLException; +import java.sql.SQLXML; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.Calendar; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; + +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.ClickHouseConfig; +import com.clickhouse.client.ClickHouseRecord; +import com.clickhouse.client.ClickHouseResponse; +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.ClickHouseValue; + +public class ClickHouseResultSet extends AbstractResultSet { + private ClickHouseRecord currentRow; + private Iterator rowCursor; + private int rowNumber; + private int lastReadColumn; // 1-based + private int fetchSize; + + protected final String database; + protected final String table; + protected final ClickHouseStatement statement; + protected final ClickHouseResponse response; + + protected final ClickHouseConfig config; + protected final List columns; + protected final TimeZone tsTimeZone; + protected final TimeZone dateTimeZone; + protected final int maxRows; + protected final ClickHouseResultSetMetaData metaData; + + // only for testing purpose + ClickHouseResultSet(String database, String table, ClickHouseResponse response) { + this.database = database; + this.table = table; + this.statement = null; + this.response = response; + + this.config = null; + this.tsTimeZone = TimeZone.getDefault(); + this.dateTimeZone = this.tsTimeZone; + + this.currentRow = null; + try { + this.columns = response.getColumns(); + this.metaData = new ClickHouseResultSetMetaData(database, table, columns); + + this.rowCursor = response.records().iterator(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + + this.rowNumber = 0; // before the first row + this.lastReadColumn = 0; + + this.maxRows = 0; + this.fetchSize = 0; + } + + public ClickHouseResultSet(String database, String table, ClickHouseStatement statement, + ClickHouseResponse response) throws SQLException { + if (database == null || table == null || statement == null || response == null) { + throw new IllegalArgumentException("Non-null database, table, statement, and response are required"); + } + + this.database = database; + this.table = table; + this.statement = statement; + this.response = response; + + ClickHouseConnection conn = statement.getConnection(); + this.config = statement.getConfig(); + this.tsTimeZone = conn.getEffectiveTimeZone(); + this.dateTimeZone = this.tsTimeZone; + + this.currentRow = null; + try { + this.columns = response.getColumns(); + this.metaData = new ClickHouseResultSetMetaData(database, table, columns); + + this.rowCursor = response.records().iterator(); + } catch (Exception e) { + throw SqlExceptionUtils.handle(e); + } + + this.rowNumber = 0; // before the first row + this.lastReadColumn = 0; + + this.maxRows = statement.getMaxRows(); + this.fetchSize = statement.getFetchSize(); + } + + protected void ensureRead(int columnIndex) throws SQLException { + ensureOpen(); + + if (currentRow == null) { + throw new SQLException("No data available for reading", SqlExceptionUtils.SQL_STATE_NO_DATA); + } else if (columnIndex < 1 || columnIndex > columns.size()) { + throw SqlExceptionUtils.clientError(ClickHouseUtils + .format("Column index must between 1 and %d but we got %d", columns.size(), columnIndex)); + } + } + + // this method is mocked in a test, do not make it final :-) + protected List getColumns() { + return metaData.getColumns(); + } + + protected ClickHouseValue getValue(int columnIndex) throws SQLException { + ensureRead(columnIndex); + + ClickHouseValue v = currentRow.getValue(columnIndex - 1); + lastReadColumn = columnIndex; + return v; + } + + /** + * Check if there is another row. + * + * @return {@code true} if this result set has another row after the current + * cursor position, {@code false} else + * @throws SQLException if something goes wrong + */ + protected boolean hasNext() throws SQLException { + try { + return (maxRows == 0 || rowNumber < maxRows) && rowCursor.hasNext(); + } catch (Exception e) { + throw SqlExceptionUtils.handle(e); + } + } + + public BigInteger getBigInteger(int columnIndex) throws SQLException { + return getValue(columnIndex).asBigInteger(); + } + + public BigInteger getBigInteger(String columnLabel) throws SQLException { + return getValue(findColumn(columnLabel)).asBigInteger(); + } + + public String[] getColumnNames() { + String[] columnNames = new String[columns.size()]; + int index = 0; + for (ClickHouseColumn c : getColumns()) { + columnNames[index++] = c.getColumnName(); + } + return columnNames; + } + + @Override + public void close() throws SQLException { + this.response.close(); + } + + @Override + public int findColumn(String columnLabel) throws SQLException { + ensureOpen(); + + if (columnLabel == null || columnLabel.isEmpty()) { + throw SqlExceptionUtils.clientError("Non-empty column label is required"); + } + + int index = 0; + for (ClickHouseColumn c : columns) { + index++; + if (columnLabel.equalsIgnoreCase(c.getColumnName())) { + return index; + } + } + + throw SqlExceptionUtils.clientError( + ClickHouseUtils.format("Column [%s] does not exist in %d columns", columnLabel, columns.size())); + } + + @Override + public Array getArray(int columnIndex) throws SQLException { + return new ClickHouseArray(this, columnIndex); + } + + @Override + public Array getArray(String columnLabel) throws SQLException { + return new ClickHouseArray(this, findColumn(columnLabel)); + } + + @Override + public InputStream getAsciiStream(int columnIndex) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public InputStream getAsciiStream(String columnLabel) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public BigDecimal getBigDecimal(int columnIndex) throws SQLException { + return getValue(columnIndex).asBigDecimal(); + } + + @Override + public BigDecimal getBigDecimal(String columnLabel) throws SQLException { + return getValue(findColumn(columnLabel)).asBigDecimal(); + } + + @Override + public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { + return getValue(columnIndex).asBigDecimal(scale); + } + + @Override + public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { + return getValue(findColumn(columnLabel)).asBigDecimal(scale); + } + + @Override + public InputStream getBinaryStream(int columnIndex) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public InputStream getBinaryStream(String columnLabel) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Blob getBlob(int columnIndex) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Blob getBlob(String columnLabel) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean getBoolean(int columnIndex) throws SQLException { + return getValue(columnIndex).asBoolean(); + } + + @Override + public boolean getBoolean(String columnLabel) throws SQLException { + return getValue(findColumn(columnLabel)).asBoolean(); + } + + @Override + public byte getByte(int columnIndex) throws SQLException { + return getValue(columnIndex).asByte(); + } + + @Override + public byte getByte(String columnLabel) throws SQLException { + return getValue(findColumn(columnLabel)).asByte(); + } + + @Override + public byte[] getBytes(int columnIndex) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public byte[] getBytes(String columnLabel) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Reader getCharacterStream(int columnIndex) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Reader getCharacterStream(String columnLabel) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Clob getClob(int columnIndex) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Clob getClob(String columnLabel) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getCursorName() throws SQLException { + ensureOpen(); + + // TODO Auto-generated method stub + return null; + } + + @Override + public Date getDate(int columnIndex) throws SQLException { + LocalDate date = getValue(columnIndex).asDate(); + return date != null ? Date.valueOf(date) : null; + } + + @Override + public Date getDate(String columnLabel) throws SQLException { + LocalDate date = getValue(findColumn(columnLabel)).asDate(); + return date != null ? Date.valueOf(date) : null; + } + + @Override + public Date getDate(int columnIndex, Calendar cal) throws SQLException { + + // TODO Auto-generated method stub + return null; + } + + @Override + public Date getDate(String columnLabel, Calendar cal) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public double getDouble(int columnIndex) throws SQLException { + return getValue(columnIndex).asDouble(); + } + + @Override + public double getDouble(String columnLabel) throws SQLException { + return getValue(findColumn(columnLabel)).asDouble(); + } + + @Override + public int getFetchSize() throws SQLException { + ensureOpen(); + + return fetchSize; + } + + @Override + public float getFloat(int columnIndex) throws SQLException { + return getValue(columnIndex).asFloat(); + } + + @Override + public float getFloat(String columnLabel) throws SQLException { + return getValue(findColumn(columnLabel)).asFloat(); + } + + @Override + public int getInt(int columnIndex) throws SQLException { + return getValue(columnIndex).asInteger(); + } + + @Override + public int getInt(String columnLabel) throws SQLException { + return getValue(findColumn(columnLabel)).asInteger(); + } + + @Override + public long getLong(int columnIndex) throws SQLException { + return getValue(columnIndex).asLong(); + } + + @Override + public long getLong(String columnLabel) throws SQLException { + return getValue(findColumn(columnLabel)).asLong(); + } + + @Override + public ResultSetMetaData getMetaData() throws SQLException { + ensureOpen(); + + return metaData; + } + + @Override + public Reader getNCharacterStream(int columnIndex) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Reader getNCharacterStream(String columnLabel) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public NClob getNClob(int columnIndex) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public NClob getNClob(String columnLabel) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getNString(int columnIndex) throws SQLException { + return getValue(columnIndex).asString(); + } + + @Override + public String getNString(String columnLabel) throws SQLException { + return getValue(findColumn(columnLabel)).asString(); + } + + @Override + public Object getObject(int columnIndex) throws SQLException { + ClickHouseValue v = getValue(columnIndex); + ClickHouseColumn c = columns.get(columnIndex - 1); + if (c.isArray()) { + return new ClickHouseArray(this, columnIndex); + } else if (c.isNested()) { + return new ClickHouseStruct(c.getDataType().name(), v.asArray()); + } else { + return v.asObject(); + } + } + + @Override + public Object getObject(String columnLabel) throws SQLException { + return getObject(findColumn(columnLabel)); + } + + @Override + public Object getObject(int columnIndex, Map> map) throws SQLException { + return getObject(columnIndex); + } + + @Override + public Object getObject(String columnLabel, Map> map) throws SQLException { + return getObject(columnLabel); + } + + @Override + public T getObject(int columnIndex, Class type) throws SQLException { + return getValue(columnIndex).asObject(type); + } + + @Override + public T getObject(String columnLabel, Class type) throws SQLException { + return getValue(findColumn(columnLabel)).asObject(type); + } + + @Override + public Ref getRef(int columnIndex) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("getRef not implemented"); + } + + @Override + public Ref getRef(String columnLabel) throws SQLException { + return getRef(findColumn(columnLabel)); + } + + @Override + public int getRow() throws SQLException { + ensureOpen(); + + return rowNumber; + } + + @Override + public RowId getRowId(int columnIndex) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("getRowId not implemented"); + } + + @Override + public RowId getRowId(String columnLabel) throws SQLException { + return getRowId(findColumn(columnLabel)); + } + + @Override + public SQLXML getSQLXML(int columnIndex) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils.unsupportedError("getSQLXML not implemented"); + } + + @Override + public SQLXML getSQLXML(String columnLabel) throws SQLException { + return getSQLXML(findColumn(columnLabel)); + } + + @Override + public short getShort(int columnIndex) throws SQLException { + return getValue(columnIndex).asShort(); + } + + @Override + public short getShort(String columnLabel) throws SQLException { + return getValue(findColumn(columnLabel)).asShort(); + } + + @Override + public Statement getStatement() throws SQLException { + ensureOpen(); + + return statement; + } + + @Override + public String getString(int columnIndex) throws SQLException { + return getValue(columnIndex).asString(); + } + + @Override + public String getString(String columnLabel) throws SQLException { + return getValue(findColumn(columnLabel)).asString(); + } + + @Override + public Time getTime(int columnIndex) throws SQLException { + return getTime(columnIndex, null); + } + + @Override + public Time getTime(String columnLabel) throws SQLException { + return getTime(findColumn(columnLabel)); + } + + @Override + public Time getTime(int columnIndex, Calendar cal) throws SQLException { + ClickHouseValue value = getValue(columnIndex); + if (value == null) { + return null; + } + + // unfortunately java.sql.Time does not support fractional seconds + LocalTime lt = value.asTime(); + + Time t; + if (cal == null) { + t = Time.valueOf(lt); + } else { + Calendar c = (Calendar) cal.clone(); + c.clear(); + c.set(1970, 0, 1, lt.getHour(), lt.getMinute(), lt.getSecond()); + t = new Time(c.getTimeInMillis()); + } + + return t; + } + + @Override + public Time getTime(String columnLabel, Calendar cal) throws SQLException { + return getTime(findColumn(columnLabel), cal); + } + + @Override + public Timestamp getTimestamp(int columnIndex) throws SQLException { + return getTimestamp(columnIndex, null); + } + + @Override + public Timestamp getTimestamp(String columnLabel) throws SQLException { + return getTimestamp(findColumn(columnLabel), null); + } + + @Override + public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { + ClickHouseValue value = getValue(columnIndex); + if (value == null) { + return null; + } + + ClickHouseColumn column = columns.get(columnIndex - 1); + TimeZone tz = column.getTimeZone(); + LocalDateTime dt = tz == null ? value.asDateTime(column.getScale()) + : value.asOffsetDateTime(column.getScale()).toLocalDateTime(); + + Timestamp timestamp; + if (cal == null) { + timestamp = Timestamp.valueOf(dt); + } else { + Calendar c = (Calendar) cal.clone(); + c.set(dt.getYear(), dt.getMonthValue() - 1, dt.getDayOfMonth(), dt.getHour(), dt.getMinute(), + dt.getSecond()); + timestamp = new Timestamp(c.getTimeInMillis()); + timestamp.setNanos(dt.getNano()); + } + + return timestamp; + } + + @Override + public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { + return getTimestamp(findColumn(columnLabel), cal); + } + + @Override + public URL getURL(int columnIndex) throws SQLException { + try { + return new URL(getString(columnIndex)); + } catch (MalformedURLException e) { + throw SqlExceptionUtils.clientError(e); + } + } + + @Override + public URL getURL(String columnLabel) throws SQLException { + try { + return new URL(getString(columnLabel)); + } catch (MalformedURLException e) { + throw SqlExceptionUtils.clientError(e); + } + } + + @Override + public InputStream getUnicodeStream(int columnIndex) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public InputStream getUnicodeStream(String columnLabel) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean isAfterLast() throws SQLException { + ensureOpen(); + + return currentRow == null && !hasNext(); + } + + @Override + public boolean isBeforeFirst() throws SQLException { + ensureOpen(); + + return getRow() == 0; + } + + @Override + public boolean isClosed() throws SQLException { + return response.isClosed(); + } + + @Override + public boolean isFirst() throws SQLException { + ensureOpen(); + + return getRow() == 1; + } + + @Override + public boolean isLast() throws SQLException { + ensureOpen(); + + return currentRow != null && !hasNext(); + } + + @Override + public boolean next() throws SQLException { + ensureOpen(); + + lastReadColumn = 0; + boolean hasNext = true; + if (hasNext()) { + currentRow = rowCursor.next(); + rowNumber++; + } else { + currentRow = null; + hasNext = false; + } + return hasNext; + } + + @Override + public void setFetchSize(int rows) throws SQLException { + ensureOpen(); + + this.fetchSize = rows; + } + + @Override + public boolean wasNull() throws SQLException { + ensureOpen(); + + try { + return currentRow != null && lastReadColumn > 0 && getColumns().get(lastReadColumn - 1).isNullable() + && currentRow.getValue(lastReadColumn - 1).isNullOrEmpty(); + } catch (Exception e) { + throw SqlExceptionUtils.handle(e); + } + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSetMetaData.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSetMetaData.java new file mode 100644 index 000000000..030c72d3e --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSetMetaData.java @@ -0,0 +1,147 @@ +package com.clickhouse.jdbc; + +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.List; + +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.ClickHouseUtils; + +public class ClickHouseResultSetMetaData extends Wrapper implements ResultSetMetaData { + public static ResultSetMetaData of(String database, String table, List columns) + throws SQLException { + if (database == null || table == null || columns == null) { + throw SqlExceptionUtils.clientError("Non-null database, table, and column list are required"); + } + + return new ClickHouseResultSetMetaData(database, table, columns); + } + + private final String database; + private final String table; + private final List columns; + + protected ClickHouseResultSetMetaData(String database, String table, List columns) { + this.database = database; + this.table = table; + this.columns = columns; + } + + protected List getColumns() { + return this.columns; + } + + protected ClickHouseColumn getColumn(int index) throws SQLException { + if (index < 1 || index > columns.size()) { + throw SqlExceptionUtils.clientError( + ClickHouseUtils.format("Column index must between 1 and %d but we got %d", columns.size(), index)); + } + return columns.get(index - 1); + } + + @Override + public int getColumnCount() throws SQLException { + return columns.size(); + } + + @Override + public boolean isAutoIncrement(int column) throws SQLException { + return false; + } + + @Override + public boolean isCaseSensitive(int column) throws SQLException { + return true; + } + + @Override + public boolean isSearchable(int column) throws SQLException { + return true; + } + + @Override + public boolean isCurrency(int column) throws SQLException { + return false; + } + + @Override + public int isNullable(int column) throws SQLException { + return getColumn(column).isNullable() ? columnNullable : columnNoNulls; + } + + @Override + public boolean isSigned(int column) throws SQLException { + return getColumn(column).getDataType().isSigned(); + } + + @Override + public int getColumnDisplaySize(int column) throws SQLException { + return 80; + } + + @Override + public String getColumnLabel(int column) throws SQLException { + return getColumnName(column); + } + + @Override + public String getColumnName(int column) throws SQLException { + return getColumn(column).getColumnName(); + } + + @Override + public String getSchemaName(int column) throws SQLException { + return database; + } + + @Override + public int getPrecision(int column) throws SQLException { + return getColumn(column).getPrecision(); + } + + @Override + public int getScale(int column) throws SQLException { + return getColumn(column).getScale(); + } + + @Override + public String getTableName(int column) throws SQLException { + return table; + } + + @Override + public String getCatalogName(int column) throws SQLException { + return ""; + } + + @Override + public int getColumnType(int column) throws SQLException { + return JdbcTypeMapping.toJdbcType(getColumn(column)); + } + + @Override + public String getColumnTypeName(int column) throws SQLException { + return getColumn(column).getOriginalTypeName(); + } + + @Override + public boolean isReadOnly(int column) throws SQLException { + return true; + } + + @Override + public boolean isWritable(int column) throws SQLException { + return false; + } + + @Override + public boolean isDefinitelyWritable(int column) throws SQLException { + return false; + } + + @Override + public String getColumnClassName(int column) throws SQLException { + return getColumn(column).getDataType().getObjectClass().getCanonicalName(); + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseScrollableResultSet.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseScrollableResultSet.java new file mode 100644 index 000000000..2ba831608 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseScrollableResultSet.java @@ -0,0 +1,75 @@ +package com.clickhouse.jdbc; + +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; + +import com.clickhouse.client.ClickHouseResponse; + +public class ClickHouseScrollableResultSet extends ClickHouseResultSet { + + private final List records; + + public ClickHouseScrollableResultSet(String database, String table, ClickHouseStatement statement, + ClickHouseResponse response) throws SQLException { + super(database, table, statement, response); + + this.records = new LinkedList<>(); + } + + @Override + public int getType() throws SQLException { + return TYPE_SCROLL_INSENSITIVE; + } + + @Override + public void beforeFirst() throws SQLException { + absolute(0); + } + + @Override + public void afterLast() throws SQLException { + absolute(-1); + next(); + } + + @Override + public boolean first() throws SQLException { + return absolute(1); + } + + @Override + public boolean last() throws SQLException { + return absolute(-1); + } + + @Override + public boolean absolute(int row) throws SQLException { + return false; + // TODO implemetation + /* + * if (row == 0) { rowNumber = 0; values = null; return false; } else if (row > + * 0) { if (row <= lines.size()) { rowNumber = row; values = lines.get(row - 1); + * return true; } absolute(lines.size()); while (getRow() < row && hasNext()) { + * next(); } if (row == getRow()) { return true; } else { next(); return false; + * } } else { // We have to check the number of total rows while (hasNext()) { + * next(); } if (-row > lines.size()) { // there is not so many rows // Put the + * cursor before the first row return absolute(0); } return + * absolute(lines.size() + 1 + row); } + */ + } + + @Override + public boolean relative(int rows) throws SQLException { + int r = getRow() + rows; + if (r < 0) { + r = 0; + } + return absolute(r); + } + + @Override + public boolean previous() throws SQLException { + return relative(-1); + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseStatement.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseStatement.java new file mode 100644 index 000000000..4d6b762a6 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseStatement.java @@ -0,0 +1,21 @@ +package com.clickhouse.jdbc; + +import java.sql.SQLException; +import java.sql.Statement; + +import com.clickhouse.client.ClickHouseConfig; +import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.client.ClickHouseRequest.Mutation; + +public interface ClickHouseStatement extends Statement { + @Override + ClickHouseConnection getConnection() throws SQLException; + + ClickHouseConfig getConfig(); + + ClickHouseRequest getRequest(); + + default Mutation write() { + return getRequest().write(); + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseStruct.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseStruct.java new file mode 100644 index 000000000..fc235748b --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseStruct.java @@ -0,0 +1,32 @@ +package com.clickhouse.jdbc; + +import java.sql.SQLException; +import java.sql.Struct; +import java.util.Map; + +import com.clickhouse.client.ClickHouseChecker; + +public class ClickHouseStruct implements Struct { + private final String typeName; + private final Object[] values; + + protected ClickHouseStruct(String typeName, Object[] values) { + this.typeName = ClickHouseChecker.nonNull(typeName, "SQLTypeName"); + this.values = ClickHouseChecker.nonNull(values, "values"); + } + + @Override + public String getSQLTypeName() throws SQLException { + return typeName; + } + + @Override + public Object[] getAttributes() throws SQLException { + return values; + } + + @Override + public Object[] getAttributes(Map> map) throws SQLException { + return getAttributes(); + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseXml.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseXml.java new file mode 100644 index 000000000..a18741625 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseXml.java @@ -0,0 +1,69 @@ +package com.clickhouse.jdbc; + +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Reader; +import java.io.Writer; +import java.sql.SQLException; +import java.sql.SQLXML; + +import javax.xml.transform.Result; +import javax.xml.transform.Source; + +public class ClickHouseXml implements SQLXML { + + @Override + public void free() throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public InputStream getBinaryStream() throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public OutputStream setBinaryStream() throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Reader getCharacterStream() throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Writer setCharacterStream() throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getString() throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void setString(String value) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public T getSource(Class sourceClass) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public T setResult(Class resultClass) throws SQLException { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/CombinedResultSet.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/CombinedResultSet.java new file mode 100644 index 000000000..475e71669 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/CombinedResultSet.java @@ -0,0 +1,557 @@ +package com.clickhouse.jdbc; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.net.URL; +import java.sql.Array; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.Date; +import java.sql.NClob; +import java.sql.Ref; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.RowId; +import java.sql.SQLException; +import java.sql.SQLXML; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Calendar; +import java.util.Collection; +import java.util.Map; + +/** + * Wrapper of multiple ResultSets. + */ +public class CombinedResultSet extends AbstractResultSet { + private final ResultSet[] results; + + private int nextIndex; + private ResultSet current; + private int rowNumber; + private boolean isClosed; + + protected ResultSet current() throws SQLException { + if (current == null) { + throw new SQLException("No result to access", SqlExceptionUtils.SQL_STATE_NO_DATA); + } + return current; + } + + protected boolean hasNext() throws SQLException { + if (current == null) { + return false; + } else if (current.next()) { + return true; + } else if (nextIndex >= results.length) { + return false; + } + + boolean hasNext = false; + while (nextIndex < results.length) { + if (current != null) { + current.close(); + } + + current = results[nextIndex++]; + if (current != null && current.next()) { + hasNext = true; + break; + } + } + + return hasNext; + } + + public CombinedResultSet(ResultSet... results) { + this.nextIndex = 0; + this.rowNumber = 0; + this.isClosed = false; + if (results == null || results.length == 0) { + this.results = new ResultSet[0]; + this.current = null; + } else { + this.results = results; + for (ResultSet rs : results) { + this.nextIndex++; + if (this.current == null && rs != null) { + this.current = rs; + break; + } + } + } + } + + public CombinedResultSet(Collection results) { + this.nextIndex = 0; + this.rowNumber = 0; + this.isClosed = false; + if (results == null || results.isEmpty()) { + this.results = new ResultSet[0]; + this.current = null; + } else { + int len = results.size(); + this.results = new ResultSet[len]; + int i = 0; + for (ResultSet rs : results) { + this.results[i++] = rs; + if (this.current == null && rs != null) { + this.current = rs; + this.nextIndex = i; + } + } + if (this.nextIndex == 0) { + this.nextIndex = len; + } + } + } + + @Override + public boolean next() throws SQLException { + if (hasNext()) { + this.rowNumber++; + return true; + } + + return false; + } + + @Override + public void close() throws SQLException { + for (ResultSet rs : results) { + if (rs == null) { + continue; + } + + try { + rs.close(); + } catch (Exception e) { + // ignore + } + } + + isClosed = true; + } + + @Override + public boolean wasNull() throws SQLException { + return current().wasNull(); + } + + @Override + public String getString(int columnIndex) throws SQLException { + return current().getString(columnIndex); + } + + @Override + public boolean getBoolean(int columnIndex) throws SQLException { + return current().getBoolean(columnIndex); + } + + @Override + public byte getByte(int columnIndex) throws SQLException { + return current().getByte(columnIndex); + } + + @Override + public short getShort(int columnIndex) throws SQLException { + return current().getShort(columnIndex); + } + + @Override + public int getInt(int columnIndex) throws SQLException { + return current().getInt(columnIndex); + } + + @Override + public long getLong(int columnIndex) throws SQLException { + return current().getLong(columnIndex); + } + + @Override + public float getFloat(int columnIndex) throws SQLException { + return current().getFloat(columnIndex); + } + + @Override + public double getDouble(int columnIndex) throws SQLException { + return current().getDouble(columnIndex); + } + + @Override + public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { + return current().getBigDecimal(columnIndex, scale); + } + + @Override + public byte[] getBytes(int columnIndex) throws SQLException { + return current().getBytes(columnIndex); + } + + @Override + public Date getDate(int columnIndex) throws SQLException { + return current().getDate(columnIndex); + } + + @Override + public Time getTime(int columnIndex) throws SQLException { + return current().getTime(columnIndex); + } + + @Override + public Timestamp getTimestamp(int columnIndex) throws SQLException { + return current().getTimestamp(columnIndex); + } + + @Override + public InputStream getAsciiStream(int columnIndex) throws SQLException { + return current().getAsciiStream(columnIndex); + } + + @Override + public InputStream getUnicodeStream(int columnIndex) throws SQLException { + return current().getUnicodeStream(columnIndex); + } + + @Override + public InputStream getBinaryStream(int columnIndex) throws SQLException { + return current().getBinaryStream(columnIndex); + } + + @Override + public String getString(String columnLabel) throws SQLException { + return current().getString(columnLabel); + } + + @Override + public boolean getBoolean(String columnLabel) throws SQLException { + return current().getBoolean(columnLabel); + } + + @Override + public byte getByte(String columnLabel) throws SQLException { + return current().getByte(columnLabel); + } + + @Override + public short getShort(String columnLabel) throws SQLException { + return current().getShort(columnLabel); + } + + @Override + public int getInt(String columnLabel) throws SQLException { + return current().getInt(columnLabel); + } + + @Override + public long getLong(String columnLabel) throws SQLException { + return current().getLong(columnLabel); + } + + @Override + public float getFloat(String columnLabel) throws SQLException { + return current().getFloat(columnLabel); + } + + @Override + public double getDouble(String columnLabel) throws SQLException { + return current().getDouble(columnLabel); + } + + @Override + public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { + return current().getBigDecimal(columnLabel, scale); + } + + @Override + public byte[] getBytes(String columnLabel) throws SQLException { + return current().getBytes(columnLabel); + } + + @Override + public Date getDate(String columnLabel) throws SQLException { + return current().getDate(columnLabel); + } + + @Override + public Time getTime(String columnLabel) throws SQLException { + return current().getTime(columnLabel); + } + + @Override + public Timestamp getTimestamp(String columnLabel) throws SQLException { + return current().getTimestamp(columnLabel); + } + + @Override + public InputStream getAsciiStream(String columnLabel) throws SQLException { + return current().getAsciiStream(columnLabel); + } + + @Override + public InputStream getUnicodeStream(String columnLabel) throws SQLException { + return current().getUnicodeStream(columnLabel); + } + + @Override + public InputStream getBinaryStream(String columnLabel) throws SQLException { + return current().getBinaryStream(columnLabel); + } + + @Override + public String getCursorName() throws SQLException { + return current().getCursorName(); + } + + @Override + public ResultSetMetaData getMetaData() throws SQLException { + return current().getMetaData(); + } + + @Override + public Object getObject(int columnIndex) throws SQLException { + return current().getObject(columnIndex); + } + + @Override + public Object getObject(String columnLabel) throws SQLException { + return current().getObject(columnLabel); + } + + @Override + public int findColumn(String columnLabel) throws SQLException { + return current().findColumn(columnLabel); + } + + @Override + public Reader getCharacterStream(int columnIndex) throws SQLException { + return current().getCharacterStream(columnIndex); + } + + @Override + public Reader getCharacterStream(String columnLabel) throws SQLException { + return current().getCharacterStream(columnLabel); + } + + @Override + public BigDecimal getBigDecimal(int columnIndex) throws SQLException { + return current().getBigDecimal(columnIndex); + } + + @Override + public BigDecimal getBigDecimal(String columnLabel) throws SQLException { + return current().getBigDecimal(columnLabel); + } + + @Override + public boolean isBeforeFirst() throws SQLException { + return nextIndex == 1 && current().isBeforeFirst(); + } + + @Override + public boolean isAfterLast() throws SQLException { + return nextIndex >= results.length && current().isAfterLast(); + } + + @Override + public boolean isFirst() throws SQLException { + return nextIndex == 1 && current().isFirst(); + } + + @Override + public boolean isLast() throws SQLException { + return (nextIndex >= results.length) && current().isLast(); + } + + @Override + public int getRow() throws SQLException { + return rowNumber; + } + + @Override + public void setFetchSize(int rows) throws SQLException { + ensureOpen(); + } + + @Override + public int getFetchSize() throws SQLException { + return current().getFetchSize(); + } + + @Override + public boolean rowUpdated() throws SQLException { + return current().rowUpdated(); + } + + @Override + public boolean rowInserted() throws SQLException { + return current().rowInserted(); + } + + @Override + public boolean rowDeleted() throws SQLException { + return current().rowDeleted(); + } + + @Override + public Statement getStatement() throws SQLException { + return current().getStatement(); + } + + @Override + public Object getObject(int columnIndex, Map> map) throws SQLException { + return current().getObject(columnIndex, map); + } + + @Override + public Ref getRef(int columnIndex) throws SQLException { + return current().getRef(columnIndex); + } + + @Override + public Blob getBlob(int columnIndex) throws SQLException { + return current().getBlob(columnIndex); + } + + @Override + public Clob getClob(int columnIndex) throws SQLException { + return current().getClob(columnIndex); + } + + @Override + public Array getArray(int columnIndex) throws SQLException { + return current().getArray(columnIndex); + } + + @Override + public Object getObject(String columnLabel, Map> map) throws SQLException { + return current().getObject(columnLabel, map); + } + + @Override + public Ref getRef(String columnLabel) throws SQLException { + return current().getRef(columnLabel); + } + + @Override + public Blob getBlob(String columnLabel) throws SQLException { + return current().getBlob(columnLabel); + } + + @Override + public Clob getClob(String columnLabel) throws SQLException { + return current().getClob(columnLabel); + } + + @Override + public Array getArray(String columnLabel) throws SQLException { + return current().getArray(columnLabel); + } + + @Override + public Date getDate(int columnIndex, Calendar cal) throws SQLException { + return current().getDate(columnIndex, cal); + } + + @Override + public Date getDate(String columnLabel, Calendar cal) throws SQLException { + return current().getDate(columnLabel, cal); + } + + @Override + public Time getTime(int columnIndex, Calendar cal) throws SQLException { + return current().getTime(columnIndex, cal); + } + + @Override + public Time getTime(String columnLabel, Calendar cal) throws SQLException { + return current().getTime(columnLabel, cal); + } + + @Override + public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { + return current().getTimestamp(columnIndex, cal); + } + + @Override + public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { + return current().getTimestamp(columnLabel, cal); + } + + @Override + public URL getURL(int columnIndex) throws SQLException { + return current().getURL(columnIndex); + } + + @Override + public URL getURL(String columnLabel) throws SQLException { + return current().getURL(columnLabel); + } + + @Override + public RowId getRowId(int columnIndex) throws SQLException { + return current().getRowId(columnIndex); + } + + @Override + public RowId getRowId(String columnLabel) throws SQLException { + return current().getRowId(columnLabel); + } + + @Override + public boolean isClosed() throws SQLException { + return isClosed; + } + + @Override + public NClob getNClob(int columnIndex) throws SQLException { + return current().getNClob(columnIndex); + } + + @Override + public NClob getNClob(String columnLabel) throws SQLException { + return current().getNClob(columnLabel); + } + + @Override + public SQLXML getSQLXML(int columnIndex) throws SQLException { + return getSQLXML(columnIndex); + } + + @Override + public SQLXML getSQLXML(String columnLabel) throws SQLException { + return getSQLXML(columnLabel); + } + + @Override + public String getNString(int columnIndex) throws SQLException { + return current().getNString(columnIndex); + } + + @Override + public String getNString(String columnLabel) throws SQLException { + return current().getNString(columnLabel); + } + + @Override + public Reader getNCharacterStream(int columnIndex) throws SQLException { + return current().getNCharacterStream(columnIndex); + } + + @Override + public Reader getNCharacterStream(String columnLabel) throws SQLException { + return current().getNCharacterStream(columnLabel); + } + + @Override + public T getObject(int columnIndex, Class type) throws SQLException { + return current().getObject(columnIndex, type); + } + + @Override + public T getObject(String columnLabel, Class type) throws SQLException { + return current().getObject(columnLabel, type); + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParameterizedQuery.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParameterizedQuery.java new file mode 100644 index 000000000..96254d55b --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParameterizedQuery.java @@ -0,0 +1,63 @@ +package com.clickhouse.jdbc; + +import java.util.ArrayList; +import java.util.List; + +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.ClickHouseParameterizedQuery; +import com.clickhouse.client.ClickHouseUtils; + +/** + * A parameterized query is a parsed query with parameters being extracted for + * substitution. + */ +public final class JdbcParameterizedQuery extends ClickHouseParameterizedQuery { + /** + * Creates an instance by parsing the given query. + * + * @param query non-empty SQL query + * @return parameterized query + */ + public static JdbcParameterizedQuery of(String query) { + // cache if query.length() is greater than 1024? + return new JdbcParameterizedQuery(query); + } + + private JdbcParameterizedQuery(String query) { + super(query); + } + + @Override + protected String parse() { + int paramIndex = 0; + int partIndex = 0; + int len = originalQuery.length(); + for (int i = 0; i < len; i++) { + char ch = originalQuery.charAt(i); + if (ClickHouseUtils.isQuote(ch)) { + i = ClickHouseUtils.skipQuotedString(originalQuery, i, len, ch) - 1; + } else if (ch == '?') { + int idx = ClickHouseUtils.skipContentsUntil(originalQuery, i + 2, len, '?', ':'); + if (idx < len && originalQuery.charAt(idx - 1) == ':' && originalQuery.charAt(idx) != ':') { + i = idx - 1; + } else { + addPart(originalQuery.substring(partIndex, i), paramIndex++, null); + partIndex = i + 1; + } + } else if (ch == ';') { + throw new IllegalArgumentException(ClickHouseUtils.format( + "Multi-statement query cannot be used in prepared statement. Please remove semicolon at %d and everything after it.", + i)); + } else if (i + 1 < len) { + char nextCh = originalQuery.charAt(i + 1); + if (ch == '-' && nextCh == ch) { + i = ClickHouseUtils.skipSingleLineComment(originalQuery, i + 2, len) - 1; + } else if (ch == '/' && nextCh == '*') { + i = ClickHouseUtils.skipMultiLineComment(originalQuery, i + 2, len) - 1; + } + } + } + + return partIndex < len ? originalQuery.substring(partIndex, len) : null; + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParseHandler.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParseHandler.java new file mode 100644 index 000000000..a010da0fe --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParseHandler.java @@ -0,0 +1,98 @@ +package com.clickhouse.jdbc; + +import java.util.List; +import java.util.Map; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.jdbc.parser.ClickHouseSqlStatement; +import com.clickhouse.jdbc.parser.ParseHandler; +import com.clickhouse.jdbc.parser.StatementType; + +public class JdbcParseHandler extends ParseHandler { + private static final String SETTING_MUTATIONS_SYNC = "mutations_sync"; + + public static final ParseHandler INSTANCE = new JdbcParseHandler(); + + private void addMutationSetting(String sql, StringBuilder builder, Map positions, + Map settings, int index) { + boolean hasSetting = settings != null && !settings.isEmpty(); + String setting = hasSetting ? settings.get(SETTING_MUTATIONS_SYNC) : null; + if (setting == null) { + String keyword = "SETTINGS"; + Integer settingsIndex = positions.get(keyword); + + if (settingsIndex == null) { + builder.append(sql.substring(index)).append(" SETTINGS mutations_sync=1"); + if (hasSetting) { + builder.append(','); + } + } else { + builder.append(sql.substring(index, settingsIndex)).append("SETTINGS mutations_sync=1,") + .append(sql.substring(settingsIndex + keyword.length())); + } + } else { + builder.append(sql.substring(index)); + } + } + + private ClickHouseSqlStatement handleDelete(String sql, StatementType stmtType, String cluster, String database, + String table, String input, String format, String outfile, List parameters, + Map positions, Map settings) { + StringBuilder builder = new StringBuilder(); + int index = positions.get("DELETE"); + if (index > 0) { + builder.append(sql.substring(0, index)); + } + index = positions.get("FROM"); + Integer whereIdx = positions.get("WHERE"); + if (whereIdx != null) { + builder.append("ALTER TABLE "); + if (!ClickHouseChecker.isNullOrEmpty(database)) { + builder.append('`').append(database).append('`').append('.'); + } + builder.append('`').append(table).append('`').append(" DELETE "); + addMutationSetting(sql, builder, positions, settings, whereIdx); + } else { + builder.append("TRUNCATE TABLE").append(sql.substring(index + 4)); + } + return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, format, + outfile, parameters, null, settings); + } + + private ClickHouseSqlStatement handleUpdate(String sql, StatementType stmtType, String cluster, String database, + String table, String input, String format, String outfile, List parameters, + Map positions, Map settings) { + StringBuilder builder = new StringBuilder(); + int index = positions.get("UPDATE"); + if (index > 0) { + builder.append(sql.substring(0, index)); + } + builder.append("ALTER TABLE "); + index = positions.get("SET"); + if (!ClickHouseChecker.isNullOrEmpty(database)) { + builder.append('`').append(database).append('`').append('.'); + } + builder.append('`').append(table).append('`').append(" UPDATE"); // .append(sql.substring(index + 3)); + addMutationSetting(sql, builder, positions, settings, index + 3); + return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, format, + outfile, parameters, null, settings); + } + + @Override + public ClickHouseSqlStatement handleStatement(String sql, StatementType stmtType, String cluster, String database, + String table, String input, String format, String outfile, List parameters, + Map positions, Map settings) { + ClickHouseSqlStatement s = null; + if (stmtType == StatementType.DELETE) { + s = handleDelete(sql, stmtType, cluster, database, table, input, format, outfile, parameters, positions, + settings); + } else if (stmtType == StatementType.UPDATE) { + s = handleUpdate(sql, stmtType, cluster, database, table, input, format, outfile, parameters, positions, + settings); + } + return s; + } + + private JdbcParseHandler() { + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcTypeMapping.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcTypeMapping.java new file mode 100644 index 000000000..63dacd2ab --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcTypeMapping.java @@ -0,0 +1,209 @@ +package com.clickhouse.jdbc; + +import java.sql.Types; + +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.ClickHouseDataType; + +public final class JdbcTypeMapping { + /** + * Gets corresponding JDBC type for the given column. + * + * @param column non-null column definition + * @return JDBC type + */ + public static int toJdbcType(ClickHouseColumn column) { + int sqlType = Types.OTHER; + + switch (column.getDataType()) { + case Enum: + case Enum8: + case Int8: + sqlType = Types.TINYINT; + break; + case UInt8: + case Enum16: + case Int16: + sqlType = Types.SMALLINT; + break; + case UInt16: + case Int32: + sqlType = Types.INTEGER; + break; + case UInt32: + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Int64: + sqlType = Types.BIGINT; + break; + case UInt64: + case Int128: + case UInt128: + case Int256: + case UInt256: + sqlType = Types.NUMERIC; + break; + case Float32: + sqlType = Types.FLOAT; + break; + case Float64: + sqlType = Types.DOUBLE; + break; + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + sqlType = Types.DECIMAL; + break; + case Date: + case Date32: + sqlType = Types.DATE; + break; + case DateTime: + case DateTime32: + case DateTime64: + sqlType = column.getTimeZone() != null ? Types.TIMESTAMP_WITH_TIMEZONE : Types.TIMESTAMP; + break; + case IPv4: + case IPv6: + case FixedString: + case String: + case UUID: + sqlType = Types.VARCHAR; + break; + case Point: + case Ring: + case Polygon: + case MultiPolygon: + case Array: + sqlType = Types.ARRAY; + break; + case Nested: + sqlType = Types.STRUCT; + break; + case Nothing: + sqlType = Types.NULL; + break; + case Map: + case Tuple: + default: + break; + } + + return sqlType; + } + + public static ClickHouseColumn fromJdbcType(int jdbcType, int scaleOrLength) { + ClickHouseDataType dataType = fromJdbcType(jdbcType); + ClickHouseColumn column = null; + if (scaleOrLength > 0) { + if (jdbcType == Types.NUMERIC || jdbcType == Types.DECIMAL) { + for (ClickHouseDataType t : new ClickHouseDataType[] {}) { + if (scaleOrLength <= t.getMaxScale() / 2) { + column = ClickHouseColumn.of("", t, false, t.getMaxPrecision() - t.getMaxScale(), + scaleOrLength); + break; + } + } + } else if (dataType == ClickHouseDataType.Date) { + if (scaleOrLength > 2) { + dataType = ClickHouseDataType.Date32; + } + } else if (dataType == ClickHouseDataType.DateTime) { + column = ClickHouseColumn.of("", ClickHouseDataType.DateTime64, false, 0, scaleOrLength); + } else if (dataType == ClickHouseDataType.String) { + column = ClickHouseColumn.of("", ClickHouseDataType.FixedString, false, scaleOrLength, 0); + } + } + + return column == null ? ClickHouseColumn.of("", dataType, false, false) : column; + } + + public static ClickHouseDataType fromJdbcType(int jdbcType) { + ClickHouseDataType dataType; + + switch (jdbcType) { + case Types.BIT: + case Types.BOOLEAN: + dataType = ClickHouseDataType.UInt8; + break; + case Types.TINYINT: + dataType = ClickHouseDataType.Int8; + break; + case Types.SMALLINT: + dataType = ClickHouseDataType.Int16; + break; + case Types.INTEGER: + dataType = ClickHouseDataType.Int32; + break; + case Types.BIGINT: + dataType = ClickHouseDataType.Int64; + break; + case Types.NUMERIC: + dataType = ClickHouseDataType.Int256; + break; + case Types.FLOAT: + case Types.REAL: + dataType = ClickHouseDataType.Float32; + break; + case Types.DOUBLE: + dataType = ClickHouseDataType.Float64; + break; + case Types.DECIMAL: + dataType = ClickHouseDataType.Decimal; + break; + case Types.BLOB: + case Types.BINARY: + case Types.CHAR: + case Types.CLOB: + case Types.JAVA_OBJECT: + case Types.LONGNVARCHAR: + case Types.LONGVARBINARY: + case Types.LONGVARCHAR: + case Types.NCHAR: + case Types.NCLOB: + case Types.NVARCHAR: + case Types.OTHER: + case Types.SQLXML: + case Types.VARBINARY: + case Types.VARCHAR: + dataType = ClickHouseDataType.String; + break; + case Types.DATE: + dataType = ClickHouseDataType.Date; + break; + case Types.TIME: + case Types.TIME_WITH_TIMEZONE: + case Types.TIMESTAMP: + case Types.TIMESTAMP_WITH_TIMEZONE: + dataType = ClickHouseDataType.DateTime; + break; + case Types.ARRAY: + dataType = ClickHouseDataType.Array; + break; + case Types.STRUCT: + dataType = ClickHouseDataType.Nested; + break; + case Types.DATALINK: + case Types.DISTINCT: + case Types.REF: + case Types.REF_CURSOR: + case Types.ROWID: + case Types.NULL: + default: + dataType = ClickHouseDataType.Nothing; + break; + } + return dataType; + } + + private JdbcTypeMapping() { + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/SqlExceptionUtils.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/SqlExceptionUtils.java new file mode 100644 index 000000000..10f7a7a98 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/SqlExceptionUtils.java @@ -0,0 +1,96 @@ +package com.clickhouse.jdbc; + +import java.net.ConnectException; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; + +import com.clickhouse.client.ClickHouseException; + +/** + * Helper class for building {@link SQLException}. + */ +public final class SqlExceptionUtils { + public static final String SQL_STATE_CLIENT_ERROR = "HY000"; + public static final String SQL_STATE_CONNECTION_EXCEPTION = "08000"; + public static final String SQL_STATE_SQL_ERROR = "07000"; + public static final String SQL_STATE_NO_DATA = "02000"; + public static final String SQL_STATE_INVALID_SCHEMA = "3F000"; + public static final String SQL_STATE_INVALID_TX_STATE = "25000"; + public static final String SQL_STATE_DATA_EXCEPTION = "22000"; + public static final String SQL_STATE_FEATURE_NOT_SUPPORTED = "0A000"; + + private SqlExceptionUtils() { + } + + // https://en.wikipedia.org/wiki/SQLSTATE + private static String toSqlState(ClickHouseException e) { + String sqlState; + if (e.getErrorCode() == ClickHouseException.ERROR_NETWORK + || e.getErrorCode() == ClickHouseException.ERROR_POCO) { + sqlState = SQL_STATE_CONNECTION_EXCEPTION; + } else if (e.getErrorCode() == 0) { + sqlState = e.getCause() instanceof ConnectException ? SQL_STATE_CONNECTION_EXCEPTION + : SQL_STATE_CLIENT_ERROR; + } else { + sqlState = e.getCause() instanceof ConnectException ? SQL_STATE_CONNECTION_EXCEPTION : SQL_STATE_SQL_ERROR; + } + + return sqlState; + } + + public static SQLException clientError(String message) { + return new SQLException(message, SQL_STATE_CLIENT_ERROR, null); + } + + public static SQLException clientError(Throwable e) { + return e != null ? new SQLException(e.getMessage(), SQL_STATE_CLIENT_ERROR, e) : unknownError(); + } + + public static SQLException clientError(String message, Throwable e) { + return new SQLException(message, SQL_STATE_CLIENT_ERROR, e); + } + + public static SQLException handle(ClickHouseException e) { + return e != null ? new SQLException(e.getMessage(), toSqlState(e), e.getErrorCode(), e.getCause()) + : unknownError(); + } + + public static SQLException handle(Throwable e) { + if (e == null) { + return unknownError(); + } else if (e instanceof ClickHouseException) { + return handle((ClickHouseException) e); + } else if (e instanceof SQLException) { + return (SQLException) e; + } + + Throwable cause = e.getCause(); + if (cause instanceof ClickHouseException) { + return handle((ClickHouseException) cause); + } else if (e instanceof SQLException) { + return (SQLException) cause; + } else if (cause == null) { + cause = e; + } + + return new SQLException(cause); + } + + public static SQLException forCancellation(Exception e) { + Throwable cause = e.getCause(); + if (cause == null) { + cause = e; + } + + // operation canceled + return new SQLException(e.getMessage(), "HY008", ClickHouseException.ERROR_ABORTED, cause); + } + + public static SQLFeatureNotSupportedException unsupportedError(String message) { + return new SQLFeatureNotSupportedException(message, SQL_STATE_FEATURE_NOT_SUPPORTED); + } + + public static SQLException unknownError() { + return new SQLException("Unknown error", SQL_STATE_CLIENT_ERROR); + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/Wrapper.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/Wrapper.java new file mode 100644 index 000000000..e37dbe76e --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/Wrapper.java @@ -0,0 +1,17 @@ +package com.clickhouse.jdbc; + +import java.sql.SQLException; + +public abstract class Wrapper { + public T unwrap(Class iface) throws SQLException { + if (iface.isAssignableFrom(getClass())) { + return iface.cast(this); + } + + throw SqlExceptionUtils.unsupportedError("Cannot unwrap to " + iface.getName()); + } + + public boolean isWrapperFor(Class iface) throws SQLException { + return iface.isAssignableFrom(getClass()); + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java new file mode 100644 index 000000000..d355d7dce --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java @@ -0,0 +1,796 @@ +package com.clickhouse.jdbc.internal; + +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.Array; +import java.sql.Blob; +import java.sql.ClientInfoStatus; +import java.sql.Clob; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.NClob; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLClientInfoException; +import java.sql.SQLException; +import java.sql.SQLWarning; +import java.sql.SQLXML; +import java.sql.Savepoint; +import java.sql.Struct; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.TimeZone; +import java.util.UUID; +import java.util.concurrent.CancellationException; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseClient; +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.ClickHouseConfig; +import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseNodeSelector; +import com.clickhouse.client.ClickHouseRecord; +import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.client.ClickHouseResponse; +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.client.ClickHouseVersion; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.http.config.ClickHouseHttpOption; +import com.clickhouse.client.logging.Logger; +import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.jdbc.ClickHouseBlob; +import com.clickhouse.jdbc.ClickHouseClob; +import com.clickhouse.jdbc.ClickHouseConnection; +import com.clickhouse.jdbc.ClickHouseDatabaseMetaData; +import com.clickhouse.jdbc.ClickHouseStatement; +import com.clickhouse.jdbc.ClickHouseXml; +import com.clickhouse.jdbc.JdbcParameterizedQuery; +import com.clickhouse.jdbc.SqlExceptionUtils; +import com.clickhouse.jdbc.Wrapper; +import com.clickhouse.jdbc.internal.ClickHouseJdbcUrlParser.ConnectionInfo; +import com.clickhouse.jdbc.internal.FakeTransaction.FakeSavepoint; + +public class ClickHouseConnectionImpl extends Wrapper implements ClickHouseConnection { + private static final Logger log = LoggerFactory.getLogger(ClickHouseConnectionImpl.class); + + // The name of the application currently utilizing the connection + private static final String PROP_APPLICATION_NAME = "ApplicationName"; + private static final String PROP_CUSTOM_HTTP_HEADERS = "CustomHttpHeaders"; + private static final String PROP_CUSTOM_HTTP_PARAMS = "CustomHttpParameters"; + // The name of the user that the application using the connection is performing + // work for. This may not be the same as the user name that was used in + // establishing the connection. + // private static final String PROP_CLIENT_USER = "ClientUser"; + // The hostname of the computer the application using the connection is running + // on. + // private static final String PROP_CLIENT_HOST = "ClientHostname"; + + private final boolean jdbcCompliant; + + private final ClickHouseClient client; + private final ClickHouseRequest clientRequest; + + private boolean autoCommit; + private boolean closed; + private String database; + private boolean readOnly; + private int networkTimeout; + private int rsHoldability; + private int txIsolation; + + private final TimeZone serverTimeZone; + private final TimeZone clientTimeZone; + private final ClickHouseVersion serverVersion; + private final String user; + + private final Map> typeMap; + + private final AtomicReference fakeTransaction; + + private URI uri; + + /** + * Checks if the connection is open or not. + * + * @throws SQLException when the connection is closed + */ + protected void ensureOpen() throws SQLException { + if (closed) { + throw SqlExceptionUtils.clientError("Cannot operate on a closed connection"); + } + } + + /** + * Checks if a feature can be supported or not. + * + * @param feature non-empty feature name + * @param silent whether to show warning in log or throw unsupported exception + * @throws SQLException when the feature is not supported and silent is + * {@code false} + */ + protected void ensureSupport(String feature, boolean silent) throws SQLException { + String msg = feature + " is not supported"; + + if (isJdbcCompliant()) { + if (silent) { + log.debug("[JDBC Compliant Mode] %s. Change %s to false to throw SQLException instead.", msg, + ClickHouseJdbcUrlParser.PROP_JDBC_COMPLIANT); + } else { + log.warn("[JDBC Compliant Mode] %s. Change %s to false to throw SQLException instead.", msg, + ClickHouseJdbcUrlParser.PROP_JDBC_COMPLIANT); + } + } else if (!silent) { + throw SqlExceptionUtils.unsupportedError(msg); + } + } + + protected void ensureTransactionSupport() throws SQLException { + ensureSupport("Transaction", false); + } + + // for testing only + final FakeTransaction getTransaction() { + return fakeTransaction.get(); + } + + public ClickHouseConnectionImpl(String url) throws SQLException { + this(url, new Properties()); + } + + public ClickHouseConnectionImpl(String url, Properties properties) throws SQLException { + ConnectionInfo connInfo; + try { + connInfo = ClickHouseJdbcUrlParser.parse(url, properties); + } catch (URISyntaxException | IllegalArgumentException e) { + throw SqlExceptionUtils.clientError(e); + } + + this.jdbcCompliant = Boolean.parseBoolean( + connInfo.getProperties().getProperty(ClickHouseJdbcUrlParser.PROP_JDBC_COMPLIANT, "true")); + connInfo.getProperties().remove(ClickHouseJdbcUrlParser.PROP_JDBC_COMPLIANT); + this.uri = connInfo.getUri(); + + log.debug("Creating a new connection to %s", url); + ClickHouseNode node = connInfo.getServer(); + log.debug("Target node: %s", node); + + client = ClickHouseClient.builder().options(connInfo.getProperties()) + .nodeSelector(ClickHouseNodeSelector.of(node.getProtocol())).build(); + clientRequest = client.connect(node); + ClickHouseConfig config = clientRequest.getConfig(); + String currentDb = null; + String currentUser = null; + TimeZone timeZone = null; + ClickHouseVersion version = null; + if (config.hasServerInfo()) { // when both serverTimeZone and serverVersion are configured + timeZone = config.getServerTimeZone(); + version = config.getServerVersion(); + } else { + try (ClickHouseResponse response = clientRequest.copy().option(ClickHouseClientOption.ASYNC, false) + .option(ClickHouseClientOption.COMPRESS, false).option(ClickHouseClientOption.DECOMPRESS, false) + .option(ClickHouseClientOption.FORMAT, ClickHouseFormat.RowBinaryWithNamesAndTypes) + .query("select currentDatabase(), currentUser(), timezone(), version()").execute().get()) { + ClickHouseRecord r = response.firstRecord(); + currentDb = r.getValue(0).asString(); + currentUser = r.getValue(1).asString(); + String tz = r.getValue(2).asString(); + String ver = r.getValue(3).asString(); + version = ClickHouseVersion.of(ver); + if (ClickHouseChecker.isNullOrBlank(tz)) { + tz = "UTC"; + } + // tsTimeZone.hasSameRules(ClickHouseValues.UTC_TIMEZONE) + timeZone = "UTC".equals(tz) ? ClickHouseValues.UTC_TIMEZONE : TimeZone.getTimeZone(tz); + + // update request and corresponding config + clientRequest.option(ClickHouseClientOption.SERVER_TIME_ZONE, tz) + .option(ClickHouseClientOption.SERVER_VERSION, ver); + } catch (InterruptedException | CancellationException e) { + // not going to happen as it's synchronous call + Thread.currentThread().interrupt(); + throw SqlExceptionUtils.forCancellation(e); + } catch (Exception e) { + throw SqlExceptionUtils.handle(e); + } + } + + this.autoCommit = true; + this.closed = false; + this.database = currentDb != null ? currentDb : config.getDatabase(); + this.clientRequest.use(this.database); + this.readOnly = false; + this.networkTimeout = 0; + this.rsHoldability = ResultSet.HOLD_CURSORS_OVER_COMMIT; + this.txIsolation = jdbcCompliant ? Connection.TRANSACTION_READ_COMMITTED : Connection.TRANSACTION_NONE; + + this.user = currentUser != null ? currentUser : node.getCredentials(config).getUserName(); + this.serverTimeZone = timeZone; + // cannot compare two timezone without giving a specific timestamp + this.clientTimeZone = config.isUseServerTimeZone() ? serverTimeZone + : TimeZone.getTimeZone(config.getUseTimeZone()); + this.serverVersion = version; + this.typeMap = new HashMap<>(); + this.fakeTransaction = new AtomicReference<>(); + } + + @Override + public String nativeSQL(String sql) throws SQLException { + ensureOpen(); + + // get rewritten query? + return sql; + } + + @Override + public void setAutoCommit(boolean autoCommit) throws SQLException { + ensureOpen(); + + if (this.autoCommit == autoCommit) { + return; + } + + ensureTransactionSupport(); + if (this.autoCommit = autoCommit) { // commit + FakeTransaction tx = fakeTransaction.getAndSet(null); + if (tx != null) { + tx.logTransactionDetails(log, "committed"); + tx.clear(); + } + } else { // start new transaction + if (!fakeTransaction.compareAndSet(null, new FakeTransaction())) { + log.warn("[JDBC Compliant Mode] not able to start a new transaction, reuse the exist one"); + } + } + } + + @Override + public boolean getAutoCommit() throws SQLException { + ensureOpen(); + + return autoCommit; + } + + @Override + public void commit() throws SQLException { + ensureOpen(); + + if (getAutoCommit()) { + throw SqlExceptionUtils.clientError("Cannot commit in auto-commit mode"); + } + + ensureTransactionSupport(); + + FakeTransaction tx = fakeTransaction.getAndSet(new FakeTransaction()); + if (tx == null) { + // invalid transaction state + throw new SQLException("Transaction not started", SqlExceptionUtils.SQL_STATE_INVALID_TX_STATE); + } else { + tx.logTransactionDetails(log, "committed"); + tx.clear(); + } + } + + @Override + public void rollback() throws SQLException { + ensureOpen(); + + if (getAutoCommit()) { + throw SqlExceptionUtils.clientError("Cannot rollback in auto-commit mode"); + } + + ensureTransactionSupport(); + + FakeTransaction tx = fakeTransaction.getAndSet(new FakeTransaction()); + if (tx == null) { + // invalid transaction state + throw new SQLException("Transaction not started", SqlExceptionUtils.SQL_STATE_INVALID_TX_STATE); + } else { + tx.logTransactionDetails(log, "rolled back"); + tx.clear(); + } + } + + @Override + public void close() throws SQLException { + try { + this.client.close(); + } catch (Exception e) { + log.warn("Failed to close connection due to %s", e.getMessage()); + throw SqlExceptionUtils.handle(e); + } finally { + this.closed = true; + } + + FakeTransaction tx = fakeTransaction.getAndSet(null); + if (tx != null) { + tx.logTransactionDetails(log, "committed"); + tx.clear(); + } + } + + @Override + public boolean isClosed() throws SQLException { + return closed; + } + + @Override + public DatabaseMetaData getMetaData() throws SQLException { + return new ClickHouseDatabaseMetaData(this); + } + + @Override + public void setReadOnly(boolean readOnly) throws SQLException { + ensureOpen(); + + this.readOnly = readOnly; + } + + @Override + public boolean isReadOnly() throws SQLException { + ensureOpen(); + + return readOnly; + } + + @Override + public void setCatalog(String catalog) throws SQLException { + ensureOpen(); + + log.warn("ClickHouse does not support catalog, please use setSchema instead"); + } + + @Override + public String getCatalog() throws SQLException { + ensureOpen(); + + return null; + } + + @Override + public void setTransactionIsolation(int level) throws SQLException { + ensureOpen(); + + if (level == Connection.TRANSACTION_READ_UNCOMMITTED || level == Connection.TRANSACTION_READ_COMMITTED + || level == Connection.TRANSACTION_REPEATABLE_READ || level == Connection.TRANSACTION_SERIALIZABLE) { + txIsolation = level; + } else { + throw new SQLException("Invalid transaction isolation level: " + level); + } + } + + @Override + public int getTransactionIsolation() throws SQLException { + ensureOpen(); + + return txIsolation; + } + + @Override + public SQLWarning getWarnings() throws SQLException { + ensureOpen(); + + return null; + } + + @Override + public void clearWarnings() throws SQLException { + ensureOpen(); + } + + @Override + public Map> getTypeMap() throws SQLException { + ensureOpen(); + + return new HashMap<>(typeMap); + } + + @Override + public void setTypeMap(Map> map) throws SQLException { + ensureOpen(); + + if (map != null) { + typeMap.putAll(map); + } + } + + @Override + public void setHoldability(int holdability) throws SQLException { + ensureOpen(); + + if (holdability == ResultSet.CLOSE_CURSORS_AT_COMMIT || holdability == ResultSet.HOLD_CURSORS_OVER_COMMIT) { + rsHoldability = holdability; + } else { + throw new SQLException("Invalid holdability: " + holdability); + } + } + + @Override + public int getHoldability() throws SQLException { + ensureOpen(); + + return rsHoldability; + } + + @Override + public Savepoint setSavepoint() throws SQLException { + return setSavepoint(null); + } + + @Override + public Savepoint setSavepoint(String name) throws SQLException { + ensureOpen(); + + if (getAutoCommit()) { + throw SqlExceptionUtils.clientError("Cannot set savepoint in auto-commit mode"); + } + + if (!isJdbcCompliant()) { + throw SqlExceptionUtils.unsupportedError("setSavepoint not implemented"); + } + + FakeTransaction tx = fakeTransaction.updateAndGet(current -> current != null ? current : new FakeTransaction()); + return tx.newSavepoint(name); + } + + @Override + public void rollback(Savepoint savepoint) throws SQLException { + ensureOpen(); + + if (getAutoCommit()) { + throw SqlExceptionUtils.clientError("Cannot rollback to savepoint in auto-commit mode"); + } + + if (!isJdbcCompliant()) { + throw SqlExceptionUtils.unsupportedError("rollback not implemented"); + } + + if (!(savepoint instanceof FakeSavepoint)) { + throw SqlExceptionUtils.clientError("Unsupported type of savepoint: " + savepoint); + } + + FakeTransaction tx = fakeTransaction.get(); + if (tx == null) { + // invalid transaction state + throw new SQLException("Transaction not started", SqlExceptionUtils.SQL_STATE_INVALID_TX_STATE); + } else { + FakeSavepoint s = (FakeSavepoint) savepoint; + tx.logSavepointDetails(log, s, "rolled back"); + tx.toSavepoint(s); + } + } + + @Override + public void releaseSavepoint(Savepoint savepoint) throws SQLException { + ensureOpen(); + + if (getAutoCommit()) { + throw SqlExceptionUtils.clientError("Cannot release savepoint in auto-commit mode"); + } + + if (!isJdbcCompliant()) { + throw SqlExceptionUtils.unsupportedError("rollback not implemented"); + } + + if (!(savepoint instanceof FakeSavepoint)) { + throw SqlExceptionUtils.clientError("Unsupported type of savepoint: " + savepoint); + } + + FakeTransaction tx = fakeTransaction.get(); + if (tx == null) { + // invalid transaction state + throw new SQLException("Transaction not started", SqlExceptionUtils.SQL_STATE_INVALID_TX_STATE); + } else { + FakeSavepoint s = (FakeSavepoint) savepoint; + tx.logSavepointDetails(log, s, "released"); + tx.toSavepoint(s); + } + } + + @Override + public ClickHouseStatement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException { + ensureOpen(); + + return new ClickHouseStatementImpl(this, clientRequest.copy(), resultSetType, resultSetConcurrency, + resultSetHoldability); + } + + @Override + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, + int resultSetHoldability) throws SQLException { + ensureOpen(); + + JdbcParameterizedQuery preparedQuery; + try { + preparedQuery = JdbcParameterizedQuery.of(sql); + } catch (RuntimeException e) { + throw SqlExceptionUtils.clientError(e); + } + + PreparedStatement ps = null; + if (!preparedQuery.hasParameter()) { + // is it insert statement using input function? + int index = 0; + int len = sql.length(); + List columns = Collections.emptyList(); + index = ClickHouseUtils.skipContentsUntil(sql, index, len, new String[] { "insert", "into" }, false); + if (index < len) { // insert statement + index = ClickHouseUtils.skipContentsUntil(sql, index, len, new String[] { "from", "input" }, false); + List params = new ArrayList<>(); + index = ClickHouseUtils.readParameters(sql, index, len, params); + + if (params.size() == 1) { + ps = new StreamBasedPreparedStatement(this, clientRequest.write().query(sql, newQueryId()), + ClickHouseColumn.parse(ClickHouseUtils.unescape(params.get(0))), resultSetType, + resultSetConcurrency, resultSetHoldability); + } + } + } + + return ps != null ? ps + : new SqlBasedPreparedStatement(this, clientRequest.copy().query(preparedQuery, newQueryId()), + preparedQuery, resultSetType, resultSetConcurrency, resultSetHoldability); + } + + @Override + public Clob createClob() throws SQLException { + return createNClob(); + } + + @Override + public Blob createBlob() throws SQLException { + ensureOpen(); + + return new ClickHouseBlob(); + } + + @Override + public NClob createNClob() throws SQLException { + ensureOpen(); + + return new ClickHouseClob(); + } + + @Override + public SQLXML createSQLXML() throws SQLException { + ensureOpen(); + + return new ClickHouseXml(); + } + + @Override + public boolean isValid(int timeout) throws SQLException { + if (timeout < 0) { + throw SqlExceptionUtils.clientError("Negative milliseconds is not allowed"); + } + + if (isClosed()) { + return false; + } + + return client.ping(clientRequest.getServer(), (int) TimeUnit.SECONDS.toMillis(timeout)); + } + + @Override + public void setClientInfo(String name, String value) throws SQLClientInfoException { + try { + ensureOpen(); + } catch (SQLException e) { + Map failedProps = new HashMap<>(); + failedProps.put(PROP_APPLICATION_NAME, ClientInfoStatus.REASON_UNKNOWN_PROPERTY); + failedProps.put(PROP_CUSTOM_HTTP_HEADERS, ClientInfoStatus.REASON_UNKNOWN_PROPERTY); + failedProps.put(PROP_CUSTOM_HTTP_PARAMS, ClientInfoStatus.REASON_UNKNOWN_PROPERTY); + throw new SQLClientInfoException(e.getMessage(), failedProps); + } + + if (PROP_APPLICATION_NAME.equals(name)) { + if (ClickHouseChecker.isNullOrBlank(value)) { + clientRequest.removeOption(ClickHouseClientOption.CLIENT_NAME); + } else { + clientRequest.option(ClickHouseClientOption.CLIENT_NAME, value); + } + } else if (PROP_CUSTOM_HTTP_HEADERS.equals(name)) { + if (ClickHouseChecker.isNullOrBlank(value)) { + clientRequest.removeOption(ClickHouseHttpOption.CUSTOM_HEADERS); + } else { + clientRequest.option(ClickHouseHttpOption.CUSTOM_HEADERS, value); + } + } else if (PROP_CUSTOM_HTTP_PARAMS.equals(name)) { + if (ClickHouseChecker.isNullOrBlank(value)) { + clientRequest.removeOption(ClickHouseHttpOption.CUSTOM_PARAMS); + } else { + clientRequest.option(ClickHouseHttpOption.CUSTOM_PARAMS, value); + } + } + } + + @Override + public void setClientInfo(Properties properties) throws SQLClientInfoException { + try { + ensureOpen(); + } catch (SQLException e) { + Map failedProps = new HashMap<>(); + failedProps.put(PROP_APPLICATION_NAME, ClientInfoStatus.REASON_UNKNOWN_PROPERTY); + failedProps.put(PROP_CUSTOM_HTTP_HEADERS, ClientInfoStatus.REASON_UNKNOWN_PROPERTY); + failedProps.put(PROP_CUSTOM_HTTP_PARAMS, ClientInfoStatus.REASON_UNKNOWN_PROPERTY); + throw new SQLClientInfoException(e.getMessage(), failedProps); + } + + if (properties != null) { + String value = properties.getProperty(PROP_APPLICATION_NAME); + if (ClickHouseChecker.isNullOrBlank(value)) { + clientRequest.removeOption(ClickHouseClientOption.CLIENT_NAME); + } else { + clientRequest.option(ClickHouseClientOption.CLIENT_NAME, value); + } + + value = properties.getProperty(PROP_CUSTOM_HTTP_HEADERS); + if (ClickHouseChecker.isNullOrBlank(value)) { + clientRequest.removeOption(ClickHouseHttpOption.CUSTOM_HEADERS); + } else { + clientRequest.option(ClickHouseHttpOption.CUSTOM_HEADERS, value); + } + + value = properties.getProperty(PROP_CUSTOM_HTTP_PARAMS); + if (ClickHouseChecker.isNullOrBlank(value)) { + clientRequest.removeOption(ClickHouseHttpOption.CUSTOM_PARAMS); + } else { + clientRequest.option(ClickHouseHttpOption.CUSTOM_PARAMS, value); + } + } + } + + @Override + public String getClientInfo(String name) throws SQLException { + ensureOpen(); + + ClickHouseConfig config = clientRequest.getConfig(); + String value = null; + if (PROP_APPLICATION_NAME.equals(name)) { + value = config.getClientName(); + } else if (PROP_CUSTOM_HTTP_HEADERS.equals(name)) { + value = (String) config.getOption(ClickHouseHttpOption.CUSTOM_HEADERS); + } else if (PROP_CUSTOM_HTTP_PARAMS.equals(name)) { + value = (String) config.getOption(ClickHouseHttpOption.CUSTOM_PARAMS); + } + return value; + } + + @Override + public Properties getClientInfo() throws SQLException { + ensureOpen(); + + ClickHouseConfig config = clientRequest.getConfig(); + Properties props = new Properties(); + props.setProperty(PROP_APPLICATION_NAME, config.getClientName()); + props.setProperty(PROP_CUSTOM_HTTP_HEADERS, (String) config.getOption(ClickHouseHttpOption.CUSTOM_HEADERS)); + props.setProperty(PROP_CUSTOM_HTTP_PARAMS, (String) config.getOption(ClickHouseHttpOption.CUSTOM_PARAMS)); + return props; + } + + @Override + public Array createArrayOf(String typeName, Object[] elements) throws SQLException { + // TODO Auto-generated method stub + // return new + // ClickHouseArray(ClickHouseDataType.resolveDefaultArrayDataType(typeName), + // elements); + return null; + } + + @Override + public Struct createStruct(String typeName, Object[] attributes) throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void setSchema(String schema) throws SQLException { + ensureOpen(); + + if (schema == null || schema.isEmpty()) { + throw new SQLException("Non-empty schema name is required", SqlExceptionUtils.SQL_STATE_INVALID_SCHEMA); + } else if (!schema.equals(database)) { + this.database = schema; + clientRequest.use(schema); + } + } + + @Override + public String getSchema() throws SQLException { + ensureOpen(); + + return getCurrentDatabase(); + } + + @Override + public void abort(Executor executor) throws SQLException { + if (executor == null) { + throw SqlExceptionUtils.clientError("Non-null executor is required"); + } + + executor.execute(() -> { + try { + // try harder please + this.client.close(); + } finally { + this.closed = true; + } + }); + } + + @Override + public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { + ensureOpen(); + + if (executor == null) { + throw SqlExceptionUtils.clientError("Non-null executor is required"); + } + + if (milliseconds < 0) { + throw SqlExceptionUtils.clientError("Negative milliseconds is not allowed"); + } + + executor.execute(() -> { + // TODO close this connection when any statement timed out after this amount of + // time + networkTimeout = milliseconds; + }); + } + + @Override + public int getNetworkTimeout() throws SQLException { + ensureOpen(); + + return networkTimeout; + } + + @Override + public String getCurrentDatabase() { + return database; + } + + @Override + public String getCurrentUser() { + return user; + } + + @Override + public TimeZone getEffectiveTimeZone() { + return clientTimeZone; + } + + @Override + public TimeZone getServerTimeZone() { + return serverTimeZone; + } + + @Override + public ClickHouseVersion getServerVersion() { + return serverVersion; + } + + @Override + public URI getUri() { + return uri; + } + + @Override + public boolean isJdbcCompliant() { + return jdbcCompliant; + } + + @Override + public String newQueryId() { + FakeTransaction tx = fakeTransaction.get(); + return tx != null ? tx.newQuery(null) : UUID.randomUUID().toString(); + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java new file mode 100644 index 000000000..adb436540 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java @@ -0,0 +1,211 @@ +package com.clickhouse.jdbc.internal; + +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.Properties; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseCredentials; +import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.config.ClickHouseDefaults; +import com.clickhouse.client.logging.Logger; +import com.clickhouse.client.logging.LoggerFactory; + +public class ClickHouseJdbcUrlParser { + private static final Logger log = LoggerFactory.getLogger(ClickHouseJdbcUrlParser.class); + + public static final String PROP_JDBC_COMPLIANT = "jdbc_compliant"; + + public static class ConnectionInfo { + private final URI uri; + private final ClickHouseNode server; + private final Properties props; + + protected ConnectionInfo(URI uri, ClickHouseNode server, Properties props) throws URISyntaxException { + this.uri = new URI("jdbc:clickhouse:" + server.getProtocol().name().toLowerCase(), null, server.getHost(), + server.getPort(), "/" + server.getDatabase().orElse(""), + removeCredentialsFromQuery(uri.getRawQuery()), null); + this.server = server; + this.props = props; + } + + public URI getUri() { + return uri; + } + + public ClickHouseNode getServer() { + return server; + } + + public Properties getProperties() { + return props; + } + } + + // URL pattern: + // jdbc:(clickhouse|ch)[:(grpc|http|tcp)]://host[:port][/db][?param1=value1¶m2=value2] + public static final String JDBC_PREFIX = "jdbc:"; + public static final String JDBC_CLICKHOUSE_PREFIX = JDBC_PREFIX + "clickhouse:"; + public static final String JDBC_ABBREVIATION_PREFIX = JDBC_PREFIX + "ch:"; + + private static String decode(String str) { + try { + return URLDecoder.decode(str, StandardCharsets.UTF_8.name()); + } catch (UnsupportedEncodingException e) { + // don't print the content here as it may contain password + log.warn("Failed to decode given string, fallback to the original string"); + return str; + } + } + + private static ClickHouseNode parseNode(URI uri, Properties defaults) { + ClickHouseProtocol protocol = ClickHouseProtocol.fromUriScheme(uri.getScheme()); + if (protocol == null || protocol == ClickHouseProtocol.ANY) { + throw new IllegalArgumentException("Unsupported scheme: " + uri.getScheme()); + } + + String host = uri.getHost(); + if (host == null || host.isEmpty()) { + throw new IllegalArgumentException("Host is missed or wrong"); + } + + int port = uri.getPort(); + if (port == -1) { + port = protocol.getDefaultPort(); + } + + String userName = defaults.getProperty(ClickHouseDefaults.USER.getKey()); + String password = defaults.getProperty(ClickHouseDefaults.PASSWORD.getKey()); + String userInfo = uri.getRawUserInfo(); + if (userInfo != null && !userInfo.isEmpty()) { + int index = userInfo.indexOf(':'); + if (userName == null) { + userName = index == 0 ? userName : decode(index > 0 ? userInfo.substring(0, index) : userInfo); + } + if (password == null) { + password = index < 0 ? password : decode(userInfo.substring(index + 1)); + } + } + if (userName == null || userName.isEmpty()) { + userName = (String) ClickHouseDefaults.USER.getEffectiveDefaultValue(); + } + if (password == null) { + password = (String) ClickHouseDefaults.PASSWORD.getEffectiveDefaultValue(); + } + + final String dbKey = ClickHouseDefaults.DATABASE.getKey(); + + String path = uri.getPath(); + String database; + if (path == null || path.isEmpty() || path.equals("/")) { + database = defaults.getProperty(dbKey); + } else if (path.charAt(0) == '/') { + database = defaults.getProperty(dbKey, path.substring(1)); + } else { + throw new IllegalArgumentException("wrong database name path: '" + path + "'"); + } + if (database == null || database.isEmpty()) { + database = (String) ClickHouseDefaults.DATABASE.getEffectiveDefaultValue(); + } + + defaults.setProperty(dbKey, database); + + return ClickHouseNode.builder().host(host).port(protocol, port).database(database) + .credentials(ClickHouseCredentials.fromUserAndPassword(userName, password)).build(); + } + + private static Properties parseParams(String query, Properties props) { + if (query == null || query.isEmpty()) { + return props; + } + + String[] queryKeyValues = query.split("&"); + for (String keyValue : queryKeyValues) { + int index = keyValue.indexOf('='); + if (index <= 0) { + log.warn("don't know how to handle parameter pair: %s", keyValue); + continue; + } + + props.put(decode(keyValue.substring(0, index)), decode(keyValue.substring(index + 1))); + } + return props; + } + + static String removeCredentialsFromQuery(String query) { + if (query == null || query.isEmpty()) { + return null; + } + + StringBuilder builder = new StringBuilder(query.length()); + int start = 0; + int index = 0; + do { + index = query.indexOf('&', start); + String kv = query.substring(start, index < 0 ? query.length() : index); + start += kv.length() + 1; + int i = kv.indexOf('='); + if (i > 0) { + String key = kv.substring(0, i); + if (!ClickHouseDefaults.USER.getKey().equals(key) + && !ClickHouseDefaults.PASSWORD.getKey().equals(key)) { + builder.append(kv).append('&'); + } + } + } while (index >= 0); + + if (builder.length() > 0) { + builder.setLength(builder.length() - 1); + query = builder.toString(); + } else { + query = null; + } + + return query; + } + + static Properties newProperties() { + Properties props = new Properties(); + props.setProperty(ClickHouseClientOption.ASYNC.getKey(), Boolean.FALSE.toString()); + props.setProperty(ClickHouseClientOption.FORMAT.getKey(), ClickHouseFormat.RowBinaryWithNamesAndTypes.name()); + return props; + } + + public static ConnectionInfo parse(String jdbcUrl, Properties defaults) throws URISyntaxException { + if (defaults == null) { + defaults = new Properties(); + } + + if (ClickHouseChecker.nonBlank(jdbcUrl, "JDBC URL").startsWith(JDBC_CLICKHOUSE_PREFIX)) { + jdbcUrl = jdbcUrl.substring(JDBC_CLICKHOUSE_PREFIX.length()); + } else if (jdbcUrl.startsWith(JDBC_ABBREVIATION_PREFIX)) { + jdbcUrl = jdbcUrl.substring(JDBC_ABBREVIATION_PREFIX.length()); + } else { + throw new URISyntaxException(jdbcUrl, ClickHouseUtils.format("'%s' or '%s' prefix is mandatory", + JDBC_CLICKHOUSE_PREFIX, JDBC_ABBREVIATION_PREFIX)); + } + + int index = jdbcUrl.indexOf("//"); + if (index == -1) { + throw new URISyntaxException(jdbcUrl, "Missing '//' from the given JDBC URL"); + } else if (index == 0) { + jdbcUrl = "http:" + jdbcUrl; + } + + URI uri = new URI(jdbcUrl); + Properties props = newProperties(); + props.putAll(defaults); + parseParams(uri.getQuery(), props); + return new ConnectionInfo(uri, parseNode(uri, props), props); + } + + private ClickHouseJdbcUrlParser() { + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseParameterMetaData.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseParameterMetaData.java new file mode 100644 index 000000000..2d368c76d --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseParameterMetaData.java @@ -0,0 +1,86 @@ +package com.clickhouse.jdbc.internal; + +import java.sql.ParameterMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.List; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.jdbc.JdbcTypeMapping; +import com.clickhouse.jdbc.SqlExceptionUtils; +import com.clickhouse.jdbc.Wrapper; + +public class ClickHouseParameterMetaData extends Wrapper implements ParameterMetaData { + protected final List params; + + protected ClickHouseParameterMetaData(List params) { + this.params = ClickHouseChecker.nonNull(params, "Parameters"); + } + + protected ClickHouseColumn getParameter(int param) throws SQLException { + if (param < 1 || param > params.size()) { + throw SqlExceptionUtils.clientError(ClickHouseUtils + .format("Parameter index should between 1 and %d but we got %d", params.size(), param)); + } + + return params.get(param - 1); + } + + @Override + public int getParameterCount() throws SQLException { + return params.size(); + } + + @Override + public int isNullable(int param) throws SQLException { + ClickHouseColumn p = getParameter(param); + if (p == null) { + return ParameterMetaData.parameterNullableUnknown; + } + + return p.isNullable() ? ParameterMetaData.parameterNullable : ParameterMetaData.parameterNoNulls; + } + + @Override + public boolean isSigned(int param) throws SQLException { + ClickHouseColumn p = getParameter(param); + return p != null && p.getDataType().isSigned(); + } + + @Override + public int getPrecision(int param) throws SQLException { + ClickHouseColumn p = getParameter(param); + return p != null ? p.getPrecision() : 0; + } + + @Override + public int getScale(int param) throws SQLException { + ClickHouseColumn p = getParameter(param); + return p != null ? p.getScale() : 0; + } + + @Override + public int getParameterType(int param) throws SQLException { + ClickHouseColumn p = getParameter(param); + return p != null ? JdbcTypeMapping.toJdbcType(p) : Types.OTHER; + } + + @Override + public String getParameterTypeName(int param) throws SQLException { + ClickHouseColumn p = getParameter(param); + return p != null ? p.getOriginalTypeName() : ""; + } + + @Override + public String getParameterClassName(int param) throws SQLException { + ClickHouseColumn p = getParameter(param); + return (p != null ? p.getDataType().getObjectClass() : Object.class).getName(); + } + + @Override + public int getParameterMode(int param) throws SQLException { + return ParameterMetaData.parameterModeIn; + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseStatementImpl.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseStatementImpl.java new file mode 100644 index 000000000..aa35fdb11 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseStatementImpl.java @@ -0,0 +1,628 @@ +package com.clickhouse.jdbc.internal; + +import java.io.InputStream; +import java.io.Serializable; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLWarning; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseConfig; +import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.client.ClickHouseResponse; +import com.clickhouse.client.ClickHouseResponseSummary; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.client.data.ClickHouseExternalTable; +import com.clickhouse.client.data.ClickHouseSimpleResponse; +import com.clickhouse.client.logging.Logger; +import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.jdbc.ClickHouseConnection; +import com.clickhouse.jdbc.ClickHouseResultSet; +import com.clickhouse.jdbc.ClickHouseStatement; +import com.clickhouse.jdbc.JdbcParseHandler; +import com.clickhouse.jdbc.SqlExceptionUtils; +import com.clickhouse.jdbc.Wrapper; +import com.clickhouse.jdbc.parser.ClickHouseSqlParser; +import com.clickhouse.jdbc.parser.ClickHouseSqlStatement; +import com.clickhouse.jdbc.parser.ParseHandler; + +public class ClickHouseStatementImpl extends Wrapper implements ClickHouseStatement { + private static final Logger log = LoggerFactory.getLogger(ClickHouseStatementImpl.class); + + private final ClickHouseConnection connection; + private final ClickHouseRequest request; + + private final int resultSetType; + private final int resultSetConcurrency; + private final int resultSetHoldability; + + private boolean closed; + private boolean closeOnCompletion; + + private String cursorName; + private boolean escapeScan; + private int fetchSize; + private int maxFieldSize; + private int maxRows; + private boolean poolable; + private String queryId; + private int queryTimeout; + + private ClickHouseResultSet currentResult; + private int currentUpdateCount; + + protected ClickHouseSqlStatement[] parsedStmts; + protected List batchStmts; + + private ClickHouseResponse getLastResponse(Map options, + List tables, Map settings) throws SQLException { + // disable extremes + if (parsedStmts.length > 1) { + request.session(UUID.randomUUID().toString()); + } + ClickHouseResponse response = null; + for (int i = 0, len = parsedStmts.length; i < len; i++) { + ClickHouseSqlStatement stmt = parsedStmts[i]; + // TODO skip useless queries to reduce network calls and server load + try { + response = request.query(stmt.getSQL(), queryId = connection.newQueryId()).execute().get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw SqlExceptionUtils.forCancellation(e); + } catch (Exception e) { + throw SqlExceptionUtils.handle(e); + } finally { + if (i + 1 < len && response != null) { + response.close(); + } + } + } + + return response; + } + + protected ClickHouseSqlStatement applyFormat(ClickHouseSqlStatement stmt, ClickHouseFormat preferredFormat) { + /* + * if (ClickHouseChecker.nonNull(stmt, "ParsedStatement").isQuery() && + * !stmt.hasFormat()) { String sql = stmt.getSQL(); String format = + * ClickHouseChecker.nonNull(preferredFormat, "Format").name(); + * + * Map positions = new HashMap<>(); + * positions.putAll(stmt.getPositions()); + * positions.put(ClickHouseSqlStatement.KEYWORD_FORMAT, sql.length()); + * + * sql = new StringBuilder(sql).append("\nFORMAT ").append(format).toString(); + * stmt = new ClickHouseSqlStatement(sql, stmt.getStatementType(), + * stmt.getCluster(), stmt.getDatabase(), stmt.getTable(), format, + * stmt.getOutfile(), stmt.getParameters(), positions); } + */ + + return stmt; + } + + protected void ensureOpen() throws SQLException { + if (closed) { + throw SqlExceptionUtils.clientError("Cannot operate on a closed statement"); + } + } + + protected int executeStatement(ClickHouseSqlStatement stmt, Map options, + List tables, Map settings) throws SQLException { + stmt = applyFormat(stmt, request.getFormat()); + + ClickHouseResponseSummary summary = null; + try (ClickHouseResponse resp = request.query(stmt.getSQL(), queryId = connection.newQueryId()).execute() + .get()) { + summary = resp.getSummary(); + } catch (InterruptedException e) { + log.error("can not close stream: %s", e.getMessage()); + Thread.currentThread().interrupt(); + SqlExceptionUtils.forCancellation(e); + } catch (Exception e) { + throw SqlExceptionUtils.handle(e); + } + + return summary != null ? (int) summary.getWrittenRows() : 1; + } + + protected int executeInsert(String sql, InputStream input) throws SQLException { + ClickHouseResponseSummary summary = null; + try (ClickHouseResponse resp = request.write().query(sql, queryId = connection.newQueryId()) + .format(ClickHouseFormat.RowBinary).data(input).execute() + .get()) { + summary = resp.getSummary(); + } catch (InterruptedException e) { + log.error("can not close stream: %s", e.getMessage()); + Thread.currentThread().interrupt(); + SqlExceptionUtils.forCancellation(e); + } catch (Exception e) { + throw SqlExceptionUtils.handle(e); + } + + return summary != null ? (int) summary.getWrittenRows() : 1; + } + + protected ClickHouseSqlStatement getLastStatement() { + ClickHouseSqlStatement stmt = null; + + if (parsedStmts != null && parsedStmts.length > 0) { + stmt = parsedStmts[parsedStmts.length - 1]; + } + + return ClickHouseChecker.nonNull(stmt, "ParsedStatement"); + } + + protected void setLastStatement(ClickHouseSqlStatement stmt) { + if (parsedStmts != null && parsedStmts.length > 0) { + parsedStmts[parsedStmts.length - 1] = ClickHouseChecker.nonNull(stmt, "ParsedStatement"); + } + } + + protected ClickHouseSqlStatement parseSqlStatements(String sql) { + parsedStmts = ClickHouseSqlParser.parse(sql, getConfig(), + connection.isJdbcCompliant() ? JdbcParseHandler.INSTANCE : null); + + if (parsedStmts == null || parsedStmts.length == 0) { + // should never happen + throw new IllegalArgumentException("Failed to parse given SQL: " + sql); + } + + ClickHouseSqlStatement lastStmt = getLastStatement(); + ClickHouseSqlStatement formattedStmt = applyFormat(lastStmt, request.getFormat()); + if (formattedStmt != lastStmt) { + setLastStatement(lastStmt = formattedStmt); + } + + return lastStmt; + } + + protected ResultSet updateResult(ClickHouseSqlStatement stmt, ClickHouseResponse response) throws SQLException { + ResultSet rs = null; + if (stmt.isQuery()) { + currentUpdateCount = -1; + currentResult = new ClickHouseResultSet(stmt.getDatabaseOrDefault(getConnection().getCurrentDatabase()), + stmt.getTable(), this, response); + rs = currentResult; + } else { + currentUpdateCount = 0; + response.close(); + } + + return rs; + } + + protected ClickHouseStatementImpl(ClickHouseConnection connection, ClickHouseRequest request, int resultSetType, + int resultSetConcurrency, int resultSetHoldability) throws SQLException { + if (connection == null || request == null) { + throw SqlExceptionUtils.clientError("Non-null connection and request are required"); + } + + this.connection = connection; + this.request = request; + + // TODO validate resultSet attributes + this.resultSetType = ResultSet.TYPE_FORWARD_ONLY; + this.resultSetConcurrency = ResultSet.CONCUR_READ_ONLY; + this.resultSetHoldability = ResultSet.CLOSE_CURSORS_AT_COMMIT; + + this.closed = false; + this.closeOnCompletion = true; + + this.fetchSize = 0; + this.maxFieldSize = 0; + this.maxRows = 0; + this.poolable = false; + this.queryId = null; + + this.queryTimeout = 0; + + this.currentResult = null; + this.currentUpdateCount = -1; + + this.batchStmts = new ArrayList<>(); + + ClickHouseConfig c = request.getConfig(); + setMaxRows(c.getMaxResultRows()); + setQueryTimeout(c.getMaxExecutionTime()); + } + + @Override + public ResultSet executeQuery(String sql) throws SQLException { + ensureOpen(); + // forcibly disable extremes for ResultSet queries + // additionalDBParams = importAdditionalDBParameters(additionalDBParams); + // FIXME respect the value set in additionalDBParams? + // additionalDBParams.put(ClickHouseQueryParam.EXTREMES, "0"); + + parseSqlStatements(sql); + + ClickHouseResponse response = getLastResponse(null, null, null); + + try { + return updateResult(getLastStatement(), response); + } catch (Exception e) { + if (response != null) { + response.close(); + } + + throw SqlExceptionUtils.handle(e); + } + } + + @Override + public int executeUpdate(String sql) throws SQLException { + ensureOpen(); + + parseSqlStatements(sql); + + ClickHouseResponseSummary summary = null; + try (ClickHouseResponse response = getLastResponse(null, null, null)) { + summary = response.getSummary(); + } catch (Exception e) { + log.error("can not close stream: %s", e.getMessage()); + } + + return summary != null ? (int) summary.getWrittenRows() : 1; + } + + @Override + public void close() throws SQLException { + if (currentResult != null) { + currentResult.close(); + } + + this.closed = true; + } + + @Override + public int getMaxFieldSize() throws SQLException { + ensureOpen(); + + return maxFieldSize; + } + + @Override + public void setMaxFieldSize(int max) throws SQLException { + if (max < 0) { + throw SqlExceptionUtils.clientError("Max field size cannot be set to negative number"); + } + ensureOpen(); + + maxFieldSize = max; + } + + @Override + public int getMaxRows() throws SQLException { + ensureOpen(); + + return maxRows; + } + + @Override + public void setMaxRows(int max) throws SQLException { + if (max < 0) { + throw SqlExceptionUtils.clientError("Max rows cannot be set to negative number"); + } + ensureOpen(); + + if (this.maxRows != max) { + if (max == 0) { + request.removeSetting("max_result_rows"); + request.removeSetting("result_overflow_mode"); + } else { + request.set("max_result_rows", max); + request.set("result_overflow_mode", "break"); + } + this.maxRows = max; + } + } + + @Override + public void setEscapeProcessing(boolean enable) throws SQLException { + ensureOpen(); + + this.escapeScan = enable; + } + + @Override + public int getQueryTimeout() throws SQLException { + ensureOpen(); + + return queryTimeout; + } + + @Override + public void setQueryTimeout(int seconds) throws SQLException { + if (seconds < 0) { + throw SqlExceptionUtils.clientError("Query timeout cannot be set to negative seconds"); + } + ensureOpen(); + + if (this.queryTimeout != seconds) { + if (seconds == 0) { + request.removeSetting("max_execution_time"); + } else { + request.set("max_execution_time", seconds); + } + this.queryTimeout = seconds; + } + } + + @Override + public void cancel() throws SQLException { + if (this.queryId == null || isClosed()) { + return; + } + + executeQuery(String.format("KILL QUERY WHERE query_id='%s'", queryId)); + } + + @Override + public SQLWarning getWarnings() throws SQLException { + ensureOpen(); + + return null; + } + + @Override + public void clearWarnings() throws SQLException { + ensureOpen(); + } + + @Override + public void setCursorName(String name) throws SQLException { + ensureOpen(); + + cursorName = name; + } + + @Override + public boolean execute(String sql) throws SQLException { + // currentResult is stored here. InputString and currentResult will be closed on + // this.close() + return executeQuery(sql) != null; + } + + @Override + public ResultSet getResultSet() throws SQLException { + ensureOpen(); + + return currentResult; + } + + @Override + public int getUpdateCount() throws SQLException { + ensureOpen(); + + return currentUpdateCount; + } + + @Override + public boolean getMoreResults() throws SQLException { + ensureOpen(); + + if (currentResult != null) { + currentResult.close(); + currentResult = null; + } + currentUpdateCount = -1; + return false; + } + + @Override + public void setFetchDirection(int direction) throws SQLException { + ensureOpen(); + + if (direction != ResultSet.FETCH_FORWARD) { + throw SqlExceptionUtils.unsupportedError("only FETCH_FORWARD is supported in setFetchDirection"); + } + } + + @Override + public int getFetchDirection() throws SQLException { + ensureOpen(); + + return ResultSet.FETCH_FORWARD; + } + + @Override + public void setFetchSize(int rows) throws SQLException { + if (rows < 0) { + throw SqlExceptionUtils.clientError("Fetch size cannot be negative number"); + } + + ensureOpen(); + + if (fetchSize != rows) { + fetchSize = rows; + + if (rows == 0) { + request.removeOption(ClickHouseClientOption.MAX_BUFFER_SIZE); + } else { + request.option(ClickHouseClientOption.MAX_BUFFER_SIZE, rows * 1024); + } + } + } + + @Override + public int getFetchSize() throws SQLException { + ensureOpen(); + + return fetchSize; + } + + @Override + public int getResultSetConcurrency() throws SQLException { + ensureOpen(); + + return resultSetConcurrency; + } + + @Override + public int getResultSetType() throws SQLException { + ensureOpen(); + + return resultSetType; + } + + @Override + public void addBatch(String sql) throws SQLException { + ensureOpen(); + + ParseHandler handler = null; + if (connection.isJdbcCompliant()) { + handler = JdbcParseHandler.INSTANCE; + } + for (ClickHouseSqlStatement s : ClickHouseSqlParser.parse(sql, getConfig(), handler)) { + this.batchStmts.add(s); + } + } + + @Override + public void clearBatch() throws SQLException { + ensureOpen(); + + this.batchStmts = new ArrayList<>(); + } + + @Override + public int[] executeBatch() throws SQLException { + ensureOpen(); + + int len = batchStmts.size(); + int[] results = new int[len]; + for (int i = 0; i < len; i++) { + try { + results[i] = executeStatement(batchStmts.get(i), null, null, null); + } catch (Exception e) { + results[i] = EXECUTE_FAILED; + log.error("Faled to execute task %d of %d", i + 1, len, e); + } + } + + clearBatch(); + + return results; + } + + @Override + public boolean getMoreResults(int current) throws SQLException { + ensureOpen(); + + switch (current) { + case Statement.KEEP_CURRENT_RESULT: + break; + case Statement.CLOSE_CURRENT_RESULT: + case Statement.CLOSE_ALL_RESULTS: + if (currentResult != null) { + currentResult.close(); + } + break; + default: + throw SqlExceptionUtils.clientError("Unknown statement constants: " + current); + } + return false; + } + + @Override + public ResultSet getGeneratedKeys() throws SQLException { + ensureOpen(); + + return new ClickHouseResultSet(request.getConfig().getDatabase(), "unknown", this, + ClickHouseSimpleResponse.EMPTY); + } + + @Override + public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException { + return executeUpdate(sql); + } + + @Override + public int executeUpdate(String sql, int[] columnIndexes) throws SQLException { + return executeUpdate(sql); + } + + @Override + public int executeUpdate(String sql, String[] columnNames) throws SQLException { + return executeUpdate(sql); + } + + @Override + public boolean execute(String sql, int autoGeneratedKeys) throws SQLException { + return execute(sql); + } + + @Override + public boolean execute(String sql, int[] columnIndexes) throws SQLException { + return execute(sql); + } + + @Override + public boolean execute(String sql, String[] columnNames) throws SQLException { + return execute(sql); + } + + @Override + public int getResultSetHoldability() throws SQLException { + ensureOpen(); + + return resultSetHoldability; + } + + @Override + public boolean isClosed() throws SQLException { + return closed; + } + + @Override + public void setPoolable(boolean poolable) throws SQLException { + ensureOpen(); + + this.poolable = poolable; + } + + @Override + public boolean isPoolable() throws SQLException { + ensureOpen(); + + return poolable; + } + + @Override + public void closeOnCompletion() throws SQLException { + ensureOpen(); + + closeOnCompletion = true; + } + + @Override + public boolean isCloseOnCompletion() throws SQLException { + ensureOpen(); + + return closeOnCompletion; + } + + @Override + public ClickHouseConnection getConnection() throws SQLException { + ensureOpen(); + + return connection; + } + + @Override + public ClickHouseConfig getConfig() { + return request.getConfig(); + } + + @Override + public ClickHouseRequest getRequest() { + return request; + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/FakeTransaction.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/FakeTransaction.java new file mode 100644 index 000000000..f85124a00 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/FakeTransaction.java @@ -0,0 +1,140 @@ +package com.clickhouse.jdbc.internal; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Savepoint; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; + +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.logging.Logger; +import com.clickhouse.jdbc.SqlExceptionUtils; + +public final class FakeTransaction { + static final int DEFAULT_TX_ISOLATION_LEVEL = Connection.TRANSACTION_READ_UNCOMMITTED; + + static final class FakeSavepoint implements Savepoint { + final int id; + final String name; + + FakeSavepoint(int id, String name) { + this.id = id; + this.name = name; + } + + @Override + public int getSavepointId() throws SQLException { + if (name != null) { + throw SqlExceptionUtils + .clientError("Cannot get ID from a named savepoint, use getSavepointName instead"); + } + + return id; + } + + @Override + public String getSavepointName() throws SQLException { + if (name == null) { + throw SqlExceptionUtils + .clientError("Cannot get name from an un-named savepoint, use getSavepointId instead"); + } + + return name; + } + + @Override + public String toString() { + return new StringBuilder().append("FakeSavepoint [id=").append(id).append(", name=").append(name) + .append(']').toString(); + } + } + + final String id; + + private final List queries; + private final List savepoints; + + FakeTransaction() { + this.id = UUID.randomUUID().toString(); + this.queries = new LinkedList<>(); + this.savepoints = new ArrayList<>(); + } + + synchronized List getQueries() { + return new ArrayList<>(queries); + } + + synchronized List getSavepoints() { + return new ArrayList<>(savepoints); + } + + synchronized void logSavepointDetails(Logger log, FakeSavepoint s, String action) { + log.warn( + "[JDBC Compliant Mode] Savepoint(id=%d, name=%s) of transaction [%s](%d queries & %d savepoints) is %s.", + s.id, s.name, id, queries.size(), savepoints.size(), action); + } + + synchronized void logTransactionDetails(Logger log, String action) { + log.warn("[JDBC Compliant Mode] Transaction [%s](%d queries & %d savepoints) is %s.", id, queries.size(), + savepoints.size(), action); + + log.debug(() -> { + log.debug("[JDBC Compliant Mode] Transaction [%s] is %s - begin", id, action); + int total = queries.size(); + int counter = 1; + for (String queryId : queries) { + log.debug(" '%s', -- query (%d of %d) in transaction [%s]", queryId, counter++, total, id); + } + + total = savepoints.size(); + counter = 1; + for (FakeSavepoint savepoint : savepoints) { + log.debug(" %s (%d of %d) in transaction [%s]", savepoint, counter++, total, id); + } + return ClickHouseUtils.format("[JDBC Compliant Mode] Transaction [%s] is %s - end", id, action); + }); + } + + synchronized String newQuery(String queryId) { + if (queryId == null || queries.contains(queryId)) { + queryId = UUID.randomUUID().toString(); + } + + queries.add(queryId); + + return queryId; + } + + synchronized FakeSavepoint newSavepoint(String name) { + FakeSavepoint savepoint = new FakeSavepoint(queries.size(), name); + this.savepoints.add(savepoint); + return savepoint; + } + + synchronized void toSavepoint(FakeSavepoint savepoint) throws SQLException { + boolean found = false; + Iterator it = savepoints.iterator(); + while (it.hasNext()) { + FakeSavepoint s = it.next(); + if (found) { + it.remove(); + } else if (s == savepoint) { + found = true; + it.remove(); + } + } + + if (!found) { + throw SqlExceptionUtils.clientError("Invalid savepoint: " + savepoint); + } + queries.subList(savepoint.id, queries.size()).clear(); + } + + synchronized void clear() { + this.queries.clear(); + this.savepoints.clear(); + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/SqlBasedPreparedStatement.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/SqlBasedPreparedStatement.java new file mode 100644 index 000000000..ef6f35b98 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/SqlBasedPreparedStatement.java @@ -0,0 +1,647 @@ +package com.clickhouse.jdbc.internal; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.sql.Array; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.Date; +import java.sql.NClob; +import java.sql.ParameterMetaData; +import java.sql.PreparedStatement; +import java.sql.Ref; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.RowId; +import java.sql.SQLException; +import java.sql.SQLXML; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.LinkedList; +import java.util.List; +import java.util.TimeZone; + +import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.client.data.BinaryStreamUtils; +import com.clickhouse.client.data.ClickHouseDateTimeValue; +import com.clickhouse.client.logging.Logger; +import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.jdbc.ClickHouseConnection; +import com.clickhouse.jdbc.JdbcParameterizedQuery; +import com.clickhouse.jdbc.JdbcTypeMapping; +import com.clickhouse.jdbc.SqlExceptionUtils; +import com.clickhouse.jdbc.parser.ClickHouseSqlStatement; + +public class SqlBasedPreparedStatement extends ClickHouseStatementImpl implements PreparedStatement { + private static final Logger log = LoggerFactory.getLogger(SqlBasedPreparedStatement.class); + + private final Calendar defaultCalendar; + private final JdbcParameterizedQuery preparedQuery; + private final ClickHouseValue[] templates; + private final String[] values; + private final List batch; + + protected SqlBasedPreparedStatement(ClickHouseConnection connection, ClickHouseRequest request, + JdbcParameterizedQuery preparedQuery, int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException { + super(connection, request, resultSetType, resultSetConcurrency, resultSetHoldability); + + defaultCalendar = new GregorianCalendar(); + defaultCalendar.setTimeZone(connection.getEffectiveTimeZone()); + + this.preparedQuery = preparedQuery; + + templates = new ClickHouseValue[preparedQuery.getNamedParameters().size()]; + values = new String[templates.length]; + batch = new LinkedList<>(); + } + + protected void ensureParams() throws SQLException { + List columns = new ArrayList<>(); + for (int i = 0, len = values.length; i < len; i++) { + if (values[i] == null) { + columns.add(String.valueOf(i + 1)); + } + } + + if (!columns.isEmpty()) { + throw SqlExceptionUtils.clientError(ClickHouseUtils.format("Missing parameter(s): %s", columns)); + } + } + + protected int toArrayIndex(int parameterIndex) throws SQLException { + if (parameterIndex < 1 || parameterIndex > templates.length) { + throw SqlExceptionUtils.clientError(ClickHouseUtils + .format("Parameter index must between 1 and %d but we got %d", templates.length, parameterIndex)); + } + + return parameterIndex - 1; + } + + @Override + public ResultSet executeQuery() throws SQLException { + ensureParams(); + + return executeQuery(preparedQuery.apply(values)); + } + + @Override + public int executeUpdate() throws SQLException { + ensureParams(); + + return executeUpdate(preparedQuery.apply(values)); + } + + @Override + public void setNull(int parameterIndex, int sqlType) throws SQLException { + setNull(parameterIndex, sqlType, null); + } + + @Override + public void setBoolean(int parameterIndex, boolean x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + ClickHouseValue value = templates[idx]; + if (value != null) { + value.update(x); + values[idx] = value.toSqlExpression(); + } else { + values[idx] = x ? "1" : "0"; + } + } + + @Override + public void setByte(int parameterIndex, byte x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + ClickHouseValue value = templates[idx]; + if (value != null) { + value.update(x); + values[idx] = value.toSqlExpression(); + } else { + values[idx] = String.valueOf(x); + } + } + + @Override + public void setShort(int parameterIndex, short x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + ClickHouseValue value = templates[idx]; + if (value != null) { + value.update(x); + values[idx] = value.toSqlExpression(); + } else { + values[idx] = String.valueOf(x); + } + } + + @Override + public void setInt(int parameterIndex, int x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + ClickHouseValue value = templates[idx]; + if (value != null) { + value.update(x); + values[idx] = value.toSqlExpression(); + } else { + values[idx] = String.valueOf(x); + } + } + + @Override + public void setLong(int parameterIndex, long x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + ClickHouseValue value = templates[idx]; + if (value != null) { + value.update(x); + values[idx] = value.toSqlExpression(); + } else { + values[idx] = String.valueOf(x); + } + } + + @Override + public void setFloat(int parameterIndex, float x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + ClickHouseValue value = templates[idx]; + if (value != null) { + value.update(x); + values[idx] = value.toSqlExpression(); + } else { + values[idx] = String.valueOf(x); + } + } + + @Override + public void setDouble(int parameterIndex, double x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + ClickHouseValue value = templates[idx]; + if (value != null) { + value.update(x); + values[idx] = value.toSqlExpression(); + } else { + values[idx] = String.valueOf(x); + } + } + + @Override + public void setBigDecimal(int parameterIndex, BigDecimal x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + ClickHouseValue value = templates[idx]; + if (value != null) { + value.update(x); + values[idx] = value.toSqlExpression(); + } else { + values[idx] = String.valueOf(x); + } + } + + @Override + public void setString(int parameterIndex, String x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + ClickHouseValue value = templates[idx]; + if (value != null) { + value.update(x); + values[idx] = value.toSqlExpression(); + } else { + values[idx] = ClickHouseValues.convertToQuotedString(x); + } + } + + @Override + public void setBytes(int parameterIndex, byte[] x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + ClickHouseValue value = templates[idx]; + if (value != null) { + value.update(x); + values[idx] = value.toSqlExpression(); + } else { + values[idx] = new String(x, StandardCharsets.UTF_8); + } + } + + @Override + public void setDate(int parameterIndex, Date x) throws SQLException { + setDate(parameterIndex, x, null); + } + + @Override + public void setTime(int parameterIndex, Time x) throws SQLException { + setTime(parameterIndex, x, null); + } + + @Override + public void setTimestamp(int parameterIndex, Timestamp x) throws SQLException { + setTimestamp(parameterIndex, x, null); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x, int length) throws SQLException { + String s = null; + if (x != null) { + try { + s = BinaryStreamUtils.readFixedString(x, length, StandardCharsets.US_ASCII); + } catch (Throwable e) { // IOException and potentially OOM error + throw SqlExceptionUtils.clientError(e); + } + } + + setString(parameterIndex, s); + } + + @Override + public void setUnicodeStream(int parameterIndex, InputStream x, int length) throws SQLException { + String s = null; + if (x != null) { + try { + s = BinaryStreamUtils.readFixedString(x, length, StandardCharsets.UTF_8); + } catch (Throwable e) { // IOException and potentially OOM error + throw SqlExceptionUtils.clientError(e); + } + } + + setString(parameterIndex, s); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException { + setUnicodeStream(parameterIndex, x, length); + } + + @Override + public void clearParameters() throws SQLException { + ensureOpen(); + + for (int i = 0, len = values.length; i < len; i++) { + values[i] = null; + } + } + + @Override + public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException { + setObject(parameterIndex, x, targetSqlType, 0); + } + + @Override + public void setObject(int parameterIndex, Object x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + ClickHouseValue value = templates[idx]; + if (value != null) { + value.update(x); + values[idx] = value.toSqlExpression(); + } else { + if (x instanceof ClickHouseValue) { + value = (ClickHouseValue) x; + templates[idx] = value; + values[idx] = value.toSqlExpression(); + } else { + values[idx] = ClickHouseValues.convertToSqlExpression(x); + } + } + } + + @Override + public boolean execute() throws SQLException { + ensureParams(); + + return execute(preparedQuery.apply(values)); + } + + @Override + public void addBatch(String sql) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils + .unsupportedError("addBatch(String) cannot be called in PreparedStatement or CallableStatement!"); + } + + @Override + public void addBatch() throws SQLException { + ensureOpen(); + + int len = values.length; + String[] newValues = new String[len]; + for (int i = 0; i < len; i++) { + String v = values[i]; + if (v == null) { + throw SqlExceptionUtils.clientError(ClickHouseUtils.format("Missing value for parameter #%d", i + 1)); + } else { + newValues[i] = v; + } + } + batch.add(newValues); + clearParameters(); + } + + @Override + public int[] executeBatch() throws SQLException { + ensureOpen(); + + int len = batch.size(); + int[] results = new int[len]; + int counter = 0; + for (String[] params : batch) { + try { + results[counter] = executeStatement(new ClickHouseSqlStatement(preparedQuery.apply(params)), null, null, + null); + } catch (Exception e) { + results[counter] = EXECUTE_FAILED; + log.error("Failed to execute task %d of %d", counter + 1, len, e); + } + counter++; + } + + clearBatch(); + + return results; + } + + @Override + public void clearBatch() throws SQLException { + ensureOpen(); + + this.batch.clear(); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader, int length) throws SQLException { + String s = null; + if (reader != null) { + try { + s = BinaryStreamUtils.readString(reader, length); + } catch (Throwable e) { // IOException and potentially OOM error + throw SqlExceptionUtils.clientError(e); + } + } + + setString(parameterIndex, s); + } + + @Override + public void setRef(int parameterIndex, Ref x) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setBlob(int parameterIndex, Blob x) throws SQLException { + if (x != null) { + setBinaryStream(parameterIndex, x.getBinaryStream()); + } else { + setNull(parameterIndex, Types.BINARY); + } + } + + @Override + public void setClob(int parameterIndex, Clob x) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setArray(int parameterIndex, Array x) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public ResultSetMetaData getMetaData() throws SQLException { + ResultSet currentResult = getResultSet(); + + return currentResult != null ? currentResult.getMetaData() : null; + } + + @Override + public void setDate(int parameterIndex, Date x, Calendar cal) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + if (x == null) { + values[idx] = ClickHouseValues.NULL_EXPR; + return; + } + + LocalDateTime dt = null; + if (cal != null) { + dt = x.toLocalDateTime().atZone(TimeZone.getDefault().toZoneId()) + .withZoneSameInstant(cal.getTimeZone().toZoneId()).toLocalDateTime(); + } else { + dt = x.toLocalDateTime(); + } + + ClickHouseValue value = templates[idx]; + if (value == null) { + value = ClickHouseDateTimeValue.ofNull(dt.getNano() > 0 ? 9 : 0); + } + values[idx] = value.update(dt).toSqlExpression(); + } + + @Override + public void setNull(int parameterIndex, int sqlType, String typeName) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + ClickHouseValue value = templates[idx]; + if (value != null) { + value.resetToNullOrEmpty(); + values[idx] = value.toSqlExpression(); + } else { + values[idx] = ClickHouseValues.NULL_EXPR; + } + } + + @Override + public void setURL(int parameterIndex, URL x) throws SQLException { + setString(parameterIndex, String.valueOf(x)); + } + + @Override + public ParameterMetaData getParameterMetaData() throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void setRowId(int parameterIndex, RowId x) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setRowId not implemented"); + } + + @Override + public void setNString(int parameterIndex, String value) throws SQLException { + setString(parameterIndex, value); + } + + @Override + public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException { + setCharacterStream(parameterIndex, value, length); + } + + @Override + public void setNClob(int parameterIndex, NClob value) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setNClob not implemented"); + } + + @Override + public void setClob(int parameterIndex, Reader reader, long length) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setClob not implemented"); + } + + @Override + public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setBlob not implemented"); + } + + @Override + public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException { + setClob(parameterIndex, reader, length); + } + + @Override + public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setSQLXML not implemented"); + } + + @Override + public void setObject(int parameterIndex, Object x, int targetSqlType, int scaleOrLength) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + ClickHouseValue value = templates[idx]; + if (value == null) { + value = ClickHouseValues.newValue(JdbcTypeMapping.fromJdbcType(targetSqlType, scaleOrLength)); + templates[idx] = value; + } + + value.update(x); + values[idx] = value.toSqlExpression(); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setAsciiStream not implemented"); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setBinaryStream not implemented"); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setCharacterStream not implemented"); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException { + setCharacterStream(parameterIndex, value); + } + + @Override + public void setClob(int parameterIndex, Reader reader) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setNClob(int parameterIndex, Reader reader) throws SQLException { + setClob(parameterIndex, reader); + } +} diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/StreamBasedPreparedStatement.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/StreamBasedPreparedStatement.java new file mode 100644 index 000000000..08aa9ddbe --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/StreamBasedPreparedStatement.java @@ -0,0 +1,600 @@ +package com.clickhouse.jdbc.internal; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.sql.Array; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.Date; +import java.sql.NClob; +import java.sql.ParameterMetaData; +import java.sql.PreparedStatement; +import java.sql.Ref; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.RowId; +import java.sql.SQLException; +import java.sql.SQLXML; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.GregorianCalendar; +import java.util.LinkedList; +import java.util.List; +import java.util.TimeZone; + +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.ClickHouseConfig; +import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.client.data.BinaryStreamUtils; +import com.clickhouse.client.data.ClickHousePipedStream; +import com.clickhouse.client.data.ClickHouseRowBinaryProcessor; +import com.clickhouse.client.data.ClickHouseRowBinaryProcessor.MappedFunctions; +import com.clickhouse.client.logging.Logger; +import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.jdbc.ClickHouseConnection; +import com.clickhouse.jdbc.SqlExceptionUtils; + +public class StreamBasedPreparedStatement extends ClickHouseStatementImpl implements PreparedStatement { + private static final Logger log = LoggerFactory.getLogger(StreamBasedPreparedStatement.class); + + private final Calendar defaultCalendar; + private final List columns; + private final ClickHouseValue[] values; + private final boolean[] flags; + + private ClickHousePipedStream stream; + private final List batch; + + protected StreamBasedPreparedStatement(ClickHouseConnection connection, ClickHouseRequest request, + List columns, int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException { + super(connection, request, resultSetType, resultSetConcurrency, resultSetHoldability); + + defaultCalendar = new GregorianCalendar(); + defaultCalendar.setTimeZone(connection.getEffectiveTimeZone()); + + this.columns = columns; + int size = columns.size(); + int i = 0; + values = new ClickHouseValue[size]; + for (ClickHouseColumn col : columns) { + values[i++] = ClickHouseValues.newValue(col); + } + flags = new boolean[size]; + + ClickHouseConfig config = request.getConfig(); + stream = new ClickHousePipedStream(config.getMaxBufferSize(), 0, config.getSocketTimeout()); + batch = new LinkedList<>(); + } + + protected void ensureParams() throws SQLException { + List list = new ArrayList<>(); + for (int i = 0, len = values.length; i < len; i++) { + if (!flags[i]) { + list.add(String.valueOf(i + 1)); + } + } + + if (!list.isEmpty()) { + throw SqlExceptionUtils.clientError(ClickHouseUtils.format("Missing parameter(s): %s", list)); + } + } + + protected int toArrayIndex(int parameterIndex) throws SQLException { + if (parameterIndex < 1 || parameterIndex > values.length) { + throw SqlExceptionUtils.clientError(ClickHouseUtils + .format("Parameter index must between 1 and %d but we got %d", values.length, parameterIndex)); + } + + return parameterIndex - 1; + } + + @Override + public ResultSet executeQuery() throws SQLException { + ensureParams(); + + return null; // return executeQuery(preparedQuery.apply(values)); + } + + @Override + public int executeUpdate() throws SQLException { + ensureParams(); + + addBatch(); + return executeBatch()[0]; + } + + @Override + public void setNull(int parameterIndex, int sqlType) throws SQLException { + setNull(parameterIndex, sqlType, null); + } + + @Override + public void setBoolean(int parameterIndex, boolean x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + values[idx].update(x); + flags[idx] = true; + } + + @Override + public void setByte(int parameterIndex, byte x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + values[idx].update(x); + flags[idx] = true; + } + + @Override + public void setShort(int parameterIndex, short x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + values[idx].update(x); + flags[idx] = true; + } + + @Override + public void setInt(int parameterIndex, int x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + values[idx].update(x); + flags[idx] = true; + } + + @Override + public void setLong(int parameterIndex, long x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + values[idx].update(x); + flags[idx] = true; + } + + @Override + public void setFloat(int parameterIndex, float x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + values[idx].update(x); + flags[idx] = true; + } + + @Override + public void setDouble(int parameterIndex, double x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + values[idx].update(x); + flags[idx] = true; + } + + @Override + public void setBigDecimal(int parameterIndex, BigDecimal x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + values[idx].update(x); + flags[idx] = true; + } + + @Override + public void setString(int parameterIndex, String x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + values[idx].update(x); + flags[idx] = true; + } + + @Override + public void setBytes(int parameterIndex, byte[] x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + values[idx].update(new String(x, StandardCharsets.UTF_8)); + flags[idx] = true; + } + + @Override + public void setDate(int parameterIndex, Date x) throws SQLException { + setDate(parameterIndex, x, null); + } + + @Override + public void setTime(int parameterIndex, Time x) throws SQLException { + setTime(parameterIndex, x, null); + } + + @Override + public void setTimestamp(int parameterIndex, Timestamp x) throws SQLException { + setTimestamp(parameterIndex, x, null); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x, int length) throws SQLException { + String s = null; + if (x != null) { + try { + s = BinaryStreamUtils.readFixedString(x, length, StandardCharsets.US_ASCII); + } catch (Throwable e) { // IOException and potentially OOM error + throw SqlExceptionUtils.clientError(e); + } + } + + setString(parameterIndex, s); + } + + @Override + public void setUnicodeStream(int parameterIndex, InputStream x, int length) throws SQLException { + String s = null; + if (x != null) { + try { + s = BinaryStreamUtils.readFixedString(x, length, StandardCharsets.UTF_8); + } catch (Throwable e) { // IOException and potentially OOM error + throw SqlExceptionUtils.clientError(e); + } + } + + setString(parameterIndex, s); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException { + setUnicodeStream(parameterIndex, x, length); + } + + @Override + public void clearParameters() throws SQLException { + ensureOpen(); + + ClickHouseConfig config = getConfig(); + if (stream != null) { + try { + stream.close(); + } catch (IOException e) { + // should not happen + throw SqlExceptionUtils.handle(e); + } + } + stream = new ClickHousePipedStream(config.getMaxBufferSize(), 0, config.getSocketTimeout()); + + for (int i = 0, len = values.length; i < len; i++) { + values[i].resetToNullOrEmpty(); + flags[i] = false; + } + } + + @Override + public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException { + setObject(parameterIndex, x, targetSqlType, 0); + } + + @Override + public void setObject(int parameterIndex, Object x) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + values[idx].update(x); + flags[idx] = true; + } + + @Override + public boolean execute() throws SQLException { + ensureParams(); + + return false; // execute(preparedQuery.apply(values)); + } + + @Override + public void addBatch(String sql) throws SQLException { + ensureOpen(); + + throw SqlExceptionUtils + .unsupportedError("addBatch(String) cannot be called in PreparedStatement or CallableStatement!"); + } + + @Override + public void addBatch() throws SQLException { + ensureOpen(); + + MappedFunctions functions = ClickHouseRowBinaryProcessor.getMappedFunctions(); + for (int i = 0, len = values.length; i < len; i++) { + if (!flags[i]) { + throw SqlExceptionUtils.clientError(ClickHouseUtils.format("Missing value for parameter #%d", i + 1)); + } + try { + functions.serialize(values[i], getConfig(), columns.get(i), stream); + } catch (IOException e) { + // should not happen + throw SqlExceptionUtils.handle(e); + } + } + + batch.add(stream.getInput()); + clearParameters(); + } + + @Override + public int[] executeBatch() throws SQLException { + ensureOpen(); + + int len = batch.size(); + int[] results = new int[len]; + int counter = 0; + for (InputStream input : batch) { + try { + results[counter] = executeInsert(getRequest().getStatements(false).get(0), input); + } catch (Exception e) { + results[counter] = EXECUTE_FAILED; + log.error("Failed to execute task %d of %d", counter + 1, len, e); + } + counter++; + } + + clearBatch(); + + return results; + } + + @Override + public void clearBatch() throws SQLException { + ensureOpen(); + + this.batch.clear(); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader, int length) throws SQLException { + String s = null; + if (reader != null) { + try { + s = BinaryStreamUtils.readString(reader, length); + } catch (Throwable e) { // IOException and potentially OOM error + throw SqlExceptionUtils.clientError(e); + } + } + + setString(parameterIndex, s); + } + + @Override + public void setRef(int parameterIndex, Ref x) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setBlob(int parameterIndex, Blob x) throws SQLException { + if (x != null) { + setBinaryStream(parameterIndex, x.getBinaryStream()); + } else { + setNull(parameterIndex, Types.BINARY); + } + } + + @Override + public void setClob(int parameterIndex, Clob x) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setArray(int parameterIndex, Array x) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public ResultSetMetaData getMetaData() throws SQLException { + ResultSet currentResult = getResultSet(); + + return currentResult != null ? currentResult.getMetaData() : null; + } + + @Override + public void setDate(int parameterIndex, Date x, Calendar cal) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + if (x == null) { + values[idx].resetToNullOrEmpty(); + flags[idx] = true; + return; + } + + LocalDateTime dt = null; + if (cal != null) { + dt = x.toLocalDateTime().atZone(TimeZone.getDefault().toZoneId()) + .withZoneSameInstant(cal.getTimeZone().toZoneId()).toLocalDateTime(); + } else { + dt = x.toLocalDateTime(); + } + + values[idx].update(dt); + flags[idx] = true; + } + + @Override + public void setNull(int parameterIndex, int sqlType, String typeName) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + values[idx].resetToNullOrEmpty(); + flags[idx] = true; + } + + @Override + public void setURL(int parameterIndex, URL x) throws SQLException { + setString(parameterIndex, String.valueOf(x)); + } + + @Override + public ParameterMetaData getParameterMetaData() throws SQLException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void setRowId(int parameterIndex, RowId x) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setRowId not implemented"); + } + + @Override + public void setNString(int parameterIndex, String value) throws SQLException { + setString(parameterIndex, value); + } + + @Override + public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException { + setCharacterStream(parameterIndex, value, length); + } + + @Override + public void setNClob(int parameterIndex, NClob value) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setNClob not implemented"); + } + + @Override + public void setClob(int parameterIndex, Reader reader, long length) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setClob not implemented"); + } + + @Override + public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setBlob not implemented"); + } + + @Override + public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException { + setClob(parameterIndex, reader, length); + } + + @Override + public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setSQLXML not implemented"); + } + + @Override + public void setObject(int parameterIndex, Object x, int targetSqlType, int scaleOrLength) throws SQLException { + ensureOpen(); + + int idx = toArrayIndex(parameterIndex); + values[idx].update(x); + flags[idx] = true; + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setAsciiStream not implemented"); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setBinaryStream not implemented"); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException { + ensureOpen(); + + toArrayIndex(parameterIndex); + + throw SqlExceptionUtils.unsupportedError("setCharacterStream not implemented"); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException { + setCharacterStream(parameterIndex, value); + } + + @Override + public void setClob(int parameterIndex, Reader reader) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException { + // TODO Auto-generated method stub + + } + + @Override + public void setNClob(int parameterIndex, Reader reader) throws SQLException { + setClob(parameterIndex, reader); + } +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlStatement.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/ClickHouseSqlStatement.java similarity index 69% rename from clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlStatement.java rename to clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/ClickHouseSqlStatement.java index f21d13f69..ea4292c5b 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlStatement.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/ClickHouseSqlStatement.java @@ -1,10 +1,12 @@ -package ru.yandex.clickhouse.jdbc.parser; +package com.clickhouse.jdbc.parser; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Map.Entry; public class ClickHouseSqlStatement { @@ -12,6 +14,7 @@ public class ClickHouseSqlStatement { public static final String DEFAULT_TABLE = "unknown"; public static final List DEFAULT_PARAMETERS = Collections.emptyList(); public static final Map DEFAULT_POSITIONS = Collections.emptyMap(); + public static final Map DEFAULT_SETTINGS = Collections.emptyMap(); public static final String KEYWORD_DATABASE = "DATABASE"; public static final String KEYWORD_EXISTS = "EXISTS"; @@ -25,50 +28,69 @@ public class ClickHouseSqlStatement { private final String cluster; private final String database; private final String table; + private final String input; private final String format; private final String outfile; private final List parameters; private final Map positions; + private final Map settings; public ClickHouseSqlStatement(String sql) { - this(sql, StatementType.UNKNOWN, null, null, null, null, null, null, null); + this(sql, StatementType.UNKNOWN, null, null, null, null, null, null, null, null, null); } public ClickHouseSqlStatement(String sql, StatementType stmtType) { - this(sql, stmtType, null, null, null, null, null, null, null); + this(sql, stmtType, null, null, null, null, null, null, null, null, null); } public ClickHouseSqlStatement(String sql, StatementType stmtType, String cluster, String database, String table, - String format, String outfile, List parameters, Map positions) { + String input, String format, String outfile, List parameters, Map positions, + Map settings) { this.sql = sql; this.stmtType = stmtType; this.cluster = cluster; this.database = database; this.table = table == null || table.isEmpty() ? DEFAULT_TABLE : table; + this.input = input; this.format = format; this.outfile = outfile; - if (parameters != null && parameters.size() > 0) { + if (parameters != null && !parameters.isEmpty()) { this.parameters = Collections.unmodifiableList(parameters); } else { this.parameters = DEFAULT_PARAMETERS; } - if (positions != null && positions.size() > 0) { + if (positions != null && !positions.isEmpty()) { Map p = new HashMap<>(); for (Entry e : positions.entrySet()) { String keyword = e.getKey(); Integer position = e.getValue(); if (keyword != null && position != null) { - p.put(keyword.toUpperCase(Locale.ROOT), position); + p.put(keyword, position); } } this.positions = Collections.unmodifiableMap(p); } else { this.positions = DEFAULT_POSITIONS; } + + if (settings != null && !settings.isEmpty()) { + Map s = new LinkedHashMap<>(); + for (Entry e : settings.entrySet()) { + String key = e.getKey(); + String value = e.getValue(); + + if (key != null && value != null) { + s.put(key, String.valueOf(e.getValue())); + } + } + this.settings = Collections.unmodifiableMap(s); + } else { + this.settings = DEFAULT_SETTINGS; + } } public String getSQL() { @@ -100,15 +122,15 @@ public boolean isIdemponent() { if (!result) { // try harder switch (this.stmtType) { - case ATTACH: - case CREATE: - case DETACH: - case DROP: - result = positions.containsKey(KEYWORD_EXISTS) || positions.containsKey(KEYWORD_REPLACE); - break; - - default: - break; + case ATTACH: + case CREATE: + case DETACH: + case DROP: + result = positions.containsKey(KEYWORD_EXISTS) || positions.containsKey(KEYWORD_REPLACE); + break; + + default: + break; } } @@ -143,6 +165,10 @@ public String getTable() { return this.table; } + public String getInput() { + return this.input; + } + public String getFormat() { return this.format; } @@ -167,6 +193,10 @@ public boolean hasOutfile() { return this.outfile != null && !this.outfile.isEmpty(); } + public boolean hasSettings() { + return !this.settings.isEmpty(); + } + public boolean hasWithTotals() { return this.positions.containsKey(KEYWORD_TOTALS); } @@ -202,14 +232,19 @@ public Map getPositions() { return this.positions; } + public Map getSettings() { + return this.settings; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append('[').append(stmtType.name()).append(']').append(" cluster=").append(cluster).append(", database=") - .append(database).append(", table=").append(table).append(", format=").append(format) - .append(", outfile=").append(outfile).append(", parameters=").append(parameters).append(", positions=") - .append(positions).append("\nSQL:\n").append(sql); + .append(database).append(", table=").append(table).append(", input=").append(input).append(", format=") + .append(format).append(", outfile=").append(outfile).append(", parameters=").append(parameters) + .append(", positions=").append(positions).append(", settings=").append(settings).append("\nSQL:\n") + .append(sql); return sb.toString(); } @@ -218,69 +253,36 @@ public String toString() { public int hashCode() { final int prime = 31; int result = 1; + result = prime * result + ((sql == null) ? 0 : sql.hashCode()); result = prime * result + ((cluster == null) ? 0 : cluster.hashCode()); result = prime * result + ((database == null) ? 0 : database.hashCode()); + result = prime * result + table.hashCode(); + result = prime * result + ((input == null) ? 0 : input.hashCode()); result = prime * result + ((format == null) ? 0 : format.hashCode()); result = prime * result + ((outfile == null) ? 0 : outfile.hashCode()); - result = prime * result + ((parameters == null) ? 0 : parameters.hashCode()); - result = prime * result + ((positions == null) ? 0 : positions.hashCode()); - result = prime * result + ((sql == null) ? 0 : sql.hashCode()); result = prime * result + ((stmtType == null) ? 0 : stmtType.hashCode()); - result = prime * result + ((table == null) ? 0 : table.hashCode()); + + result = prime * result + parameters.hashCode(); + result = prime * result + positions.hashCode(); + result = prime * result + settings.hashCode(); return result; } @Override public boolean equals(Object obj) { - if (this == obj) + if (this == obj) { return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) + } + + if (obj == null || getClass() != obj.getClass()) { return false; + } + ClickHouseSqlStatement other = (ClickHouseSqlStatement) obj; - if (cluster == null) { - if (other.cluster != null) - return false; - } else if (!cluster.equals(other.cluster)) - return false; - if (database == null) { - if (other.database != null) - return false; - } else if (!database.equals(other.database)) - return false; - if (format == null) { - if (other.format != null) - return false; - } else if (!format.equals(other.format)) - return false; - if (outfile == null) { - if (other.outfile != null) - return false; - } else if (!outfile.equals(other.outfile)) - return false; - if (parameters == null) { - if (other.parameters != null) - return false; - } else if (!parameters.equals(other.parameters)) - return false; - if (positions == null) { - if (other.positions != null) - return false; - } else if (!positions.equals(other.positions)) - return false; - if (sql == null) { - if (other.sql != null) - return false; - } else if (!sql.equals(other.sql)) - return false; - if (stmtType != other.stmtType) - return false; - if (table == null) { - if (other.table != null) - return false; - } else if (!table.equals(other.table)) - return false; - return true; + return stmtType == other.stmtType && Objects.equals(sql, other.sql) && Objects.equals(cluster, other.cluster) + && Objects.equals(database, other.database) && Objects.equals(table, other.table) + && Objects.equals(input, other.input) && Objects.equals(format, other.format) + && Objects.equals(outfile, other.outfile) && parameters.equals(other.parameters) + && positions.equals(other.positions) && settings.equals(other.settings); } } diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlUtils.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/ClickHouseSqlUtils.java similarity index 97% rename from clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlUtils.java rename to clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/ClickHouseSqlUtils.java index a7be146de..8b42372d6 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlUtils.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/ClickHouseSqlUtils.java @@ -1,4 +1,4 @@ -package ru.yandex.clickhouse.jdbc.parser; +package com.clickhouse.jdbc.parser; public final class ClickHouseSqlUtils { public static boolean isQuote(char ch) { diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/LanguageType.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/LanguageType.java similarity index 83% rename from clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/LanguageType.java rename to clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/LanguageType.java index c3fa2cfa6..4ff42e3f6 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/LanguageType.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/LanguageType.java @@ -1,4 +1,4 @@ -package ru.yandex.clickhouse.jdbc.parser; +package com.clickhouse.jdbc.parser; public enum LanguageType { UNKNOWN, // unknown language diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/OperationType.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/OperationType.java similarity index 57% rename from clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/OperationType.java rename to clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/OperationType.java index 4c5a2222f..56360c256 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/OperationType.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/OperationType.java @@ -1,4 +1,4 @@ -package ru.yandex.clickhouse.jdbc.parser; +package com.clickhouse.jdbc.parser; public enum OperationType { UNKNOWN, READ, WRITE diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ParseHandler.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/ParseHandler.java similarity index 83% rename from clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ParseHandler.java rename to clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/ParseHandler.java index d9d244487..30d8b2821 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ParseHandler.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/ParseHandler.java @@ -1,4 +1,4 @@ -package ru.yandex.clickhouse.jdbc.parser; +package com.clickhouse.jdbc.parser; import java.util.List; import java.util.Map; @@ -37,13 +37,16 @@ public String handleParameter(String cluster, String database, String table, int * @param database database * @param table table * @param format format + * @param input input * @param outfile outfile * @param parameters positions of parameters * @param positions keyword positions + * @param settings settings * @return sql statement, or null means no change */ public ClickHouseSqlStatement handleStatement(String sql, StatementType stmtType, String cluster, String database, - String table, String format, String outfile, List parameters, Map positions) { + String table, String input, String format, String outfile, List parameters, + Map positions, Map settings) { return null; } } diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/StatementType.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/StatementType.java similarity index 93% rename from clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/StatementType.java rename to clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/StatementType.java index 3797026d9..3649d7a6a 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/StatementType.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/parser/StatementType.java @@ -1,10 +1,10 @@ -package ru.yandex.clickhouse.jdbc.parser; +package com.clickhouse.jdbc.parser; public enum StatementType { UNKNOWN(LanguageType.UNKNOWN, OperationType.UNKNOWN, false), // unknown statement ALTER(LanguageType.DDL, OperationType.UNKNOWN, false), // alter statement - ALTER_DELETE(LanguageType.DDL, OperationType.WRITE, false), // delete statement - ALTER_UPDATE(LanguageType.DDL, OperationType.WRITE, false), // update statement + ALTER_DELETE(LanguageType.DML, OperationType.WRITE, false), // delete statement + ALTER_UPDATE(LanguageType.DML, OperationType.WRITE, false), // update statement ATTACH(LanguageType.DDL, OperationType.UNKNOWN, false), // attach statement CHECK(LanguageType.DDL, OperationType.UNKNOWN, true), // check statement CREATE(LanguageType.DDL, OperationType.UNKNOWN, false), // create statement diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java index 366177a73..4aa7c3a6e 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java @@ -33,12 +33,12 @@ import com.clickhouse.client.logging.Logger; import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.jdbc.parser.ClickHouseSqlStatement; +import com.clickhouse.jdbc.parser.StatementType; import org.apache.http.entity.AbstractHttpEntity; import org.apache.http.impl.client.CloseableHttpClient; -import ru.yandex.clickhouse.jdbc.parser.ClickHouseSqlStatement; -import ru.yandex.clickhouse.jdbc.parser.StatementType; import ru.yandex.clickhouse.response.ClickHouseResponse; import ru.yandex.clickhouse.settings.ClickHouseProperties; import ru.yandex.clickhouse.settings.ClickHouseQueryParam; diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java index 3b5910fc0..d9498a5a5 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java @@ -23,6 +23,9 @@ import com.clickhouse.client.data.JsonStreamUtils; import com.clickhouse.client.logging.Logger; import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.jdbc.parser.ClickHouseSqlParser; +import com.clickhouse.jdbc.parser.ClickHouseSqlStatement; +import com.clickhouse.jdbc.parser.StatementType; import org.apache.http.Header; import org.apache.http.HttpEntity; @@ -45,9 +48,6 @@ import ru.yandex.clickhouse.domain.ClickHouseFormat; import ru.yandex.clickhouse.except.ClickHouseException; import ru.yandex.clickhouse.except.ClickHouseExceptionSpecifier; -import ru.yandex.clickhouse.jdbc.parser.ClickHouseSqlParser; -import ru.yandex.clickhouse.jdbc.parser.ClickHouseSqlStatement; -import ru.yandex.clickhouse.jdbc.parser.StatementType; import ru.yandex.clickhouse.response.ClickHouseLZ4Stream; import ru.yandex.clickhouse.response.ClickHouseResponse; import ru.yandex.clickhouse.response.ClickHouseResponseSummary; @@ -99,7 +99,7 @@ public void writeTo(OutputStream outputStream) throws IOException { outputStream.write(sql.getBytes(StandardCharsets.UTF_8)); outputStream.write('\n'); } - + entity.writeTo(outputStream); } @@ -142,9 +142,10 @@ public boolean isStreaming() { protected List batchStmts; /** - * Current database name may be changed by {@link java.sql.Connection#setCatalog(String)} - * between creation of this object and query execution, but javadoc does not allow - * {@code setCatalog} influence on already created statements. + * Current database name may be changed by + * {@link java.sql.Connection#setCatalog(String)} between creation of this + * object and query execution, but javadoc does not allow {@code setCatalog} + * influence on already created statements. */ protected String currentDatabase; @@ -169,8 +170,8 @@ protected void setLastStatement(ClickHouseSqlStatement stmt) { } protected ClickHouseSqlStatement[] parseSqlStatements(String sql) throws SQLException { - parsedStmts = ClickHouseSqlParser.parse(sql, properties); - + parsedStmts = ClickHouseSqlParser.parse(sql, null); + if (parsedStmts == null || parsedStmts.length == 0) { // should never happen throw new IllegalArgumentException("Failed to parse given SQL: " + sql); @@ -179,9 +180,8 @@ protected ClickHouseSqlStatement[] parseSqlStatements(String sql) throws SQLExce return parsedStmts; } - protected ClickHouseSqlStatement parseSqlStatements( - String sql, ClickHouseFormat preferredFormat, Map additionalDBParams) - throws SQLException { + protected ClickHouseSqlStatement parseSqlStatements(String sql, ClickHouseFormat preferredFormat, + Map additionalDBParams) throws SQLException { parseSqlStatements(sql); // enable session when we have more than one statement @@ -208,15 +208,15 @@ protected ClickHouseSqlStatement applyFormat(ClickHouseSqlStatement stmt, ClickH positions.put(ClickHouseSqlStatement.KEYWORD_FORMAT, sql.length()); sql = new StringBuilder(sql).append("\nFORMAT ").append(format).toString(); - stmt = new ClickHouseSqlStatement(sql, stmt.getStatementType(), - stmt.getCluster(), stmt.getDatabase(), stmt.getTable(), - format, stmt.getOutfile(), stmt.getParameters(), positions); + stmt = new ClickHouseSqlStatement(sql, stmt.getStatementType(), stmt.getCluster(), stmt.getDatabase(), + stmt.getTable(), stmt.getInput(), format, stmt.getOutfile(), stmt.getParameters(), positions, null); } return stmt; } - protected Map importAdditionalDBParameters(Map additionalDBParams) { + protected Map importAdditionalDBParameters( + Map additionalDBParams) { if (additionalDBParams == null || additionalDBParams.isEmpty()) { additionalDBParams = new EnumMap<>(ClickHouseQueryParam.class); } else { // in case the given additionalDBParams is immutable @@ -226,19 +226,14 @@ protected Map importAdditionalDBParameters(Map additionalDBParams, - List externalData, - Map additionalRequestParams) throws SQLException { + protected int executeStatement(ClickHouseSqlStatement stmt, Map additionalDBParams, + List externalData, Map additionalRequestParams) + throws SQLException { additionalDBParams = importAdditionalDBParameters(additionalDBParams); stmt = applyFormat(stmt, ClickHouseFormat.TabSeparatedWithNamesAndTypes); try (InputStream is = getInputStream(stmt, additionalDBParams, externalData, additionalRequestParams)) { - //noinspection StatementWithEmptyBody + // noinspection StatementWithEmptyBody } catch (IOException e) { log.error("can not close stream: %s", e.getMessage()); } @@ -271,9 +264,8 @@ protected int executeStatement( } protected ResultSet executeQueryStatement(ClickHouseSqlStatement stmt, - Map additionalDBParams, - List externalData, - Map additionalRequestParams) throws SQLException { + Map additionalDBParams, List externalData, + Map additionalRequestParams) throws SQLException { additionalDBParams = importAdditionalDBParameters(additionalDBParams); stmt = applyFormat(stmt, ClickHouseFormat.TabSeparatedWithNamesAndTypes); @@ -290,23 +282,22 @@ protected ResultSet executeQueryStatement(ClickHouseSqlStatement stmt, } } - protected ClickHouseResponse executeQueryClickhouseResponse( - ClickHouseSqlStatement stmt, - Map additionalDBParams, - Map additionalRequestParams) throws SQLException { + protected ClickHouseResponse executeQueryClickhouseResponse(ClickHouseSqlStatement stmt, + Map additionalDBParams, Map additionalRequestParams) + throws SQLException { additionalDBParams = importAdditionalDBParameters(additionalDBParams); stmt = applyFormat(stmt, ClickHouseFormat.JSONCompact); - + try (InputStream is = getInputStream(stmt, additionalDBParams, null, additionalRequestParams)) { - return JsonStreamUtils.readObject( - properties.isCompress() ? new ClickHouseLZ4Stream(is) : is, ClickHouseResponse.class); + return JsonStreamUtils.readObject(properties.isCompress() ? new ClickHouseLZ4Stream(is) : is, + ClickHouseResponse.class); } catch (IOException e) { throw new RuntimeException(e); } } public ClickHouseStatementImpl(CloseableHttpClient client, ClickHouseConnection connection, - ClickHouseProperties properties, int resultSetType) { + ClickHouseProperties properties, int resultSetType) { super(null); this.client = client; this.httpContext = ClickHouseHttpClientBuilder.createClientContext(properties); @@ -324,20 +315,21 @@ public ResultSet executeQuery(String sql) throws SQLException { } @Override - public ResultSet executeQuery(String sql, Map additionalDBParams) throws SQLException { + public ResultSet executeQuery(String sql, Map additionalDBParams) + throws SQLException { return executeQuery(sql, additionalDBParams, null); } @Override - public ResultSet executeQuery(String sql, Map additionalDBParams, List externalData) throws SQLException { + public ResultSet executeQuery(String sql, Map additionalDBParams, + List externalData) throws SQLException { return executeQuery(sql, additionalDBParams, externalData, null); } @Override - public ResultSet executeQuery(String sql, - Map additionalDBParams, - List externalData, - Map additionalRequestParams) throws SQLException { + public ResultSet executeQuery(String sql, Map additionalDBParams, + List externalData, Map additionalRequestParams) + throws SQLException { // forcibly disable extremes for ResultSet queries additionalDBParams = importAdditionalDBParameters(additionalDBParams); @@ -367,20 +359,21 @@ public ClickHouseResponse executeQueryClickhouseResponse(String sql) throws SQLE } @Override - public ClickHouseResponse executeQueryClickhouseResponse(String sql, Map additionalDBParams) throws SQLException { + public ClickHouseResponse executeQueryClickhouseResponse(String sql, + Map additionalDBParams) throws SQLException { return executeQueryClickhouseResponse(sql, additionalDBParams, null); } @Override public ClickHouseResponse executeQueryClickhouseResponse(String sql, - Map additionalDBParams, - Map additionalRequestParams) throws SQLException { + Map additionalDBParams, Map additionalRequestParams) + throws SQLException { additionalDBParams = importAdditionalDBParameters(additionalDBParams); parseSqlStatements(sql, ClickHouseFormat.JSONCompact, additionalDBParams); - + try (InputStream is = getLastInputStream(additionalDBParams, null, additionalRequestParams)) { - return JsonStreamUtils.readObject( - properties.isCompress() ? new ClickHouseLZ4Stream(is) : is, ClickHouseResponse.class); + return JsonStreamUtils.readObject(properties.isCompress() ? new ClickHouseLZ4Stream(is) : is, + ClickHouseResponse.class); } catch (IOException e) { throw new RuntimeException(e); } @@ -392,28 +385,28 @@ public ClickHouseRowBinaryInputStream executeQueryClickhouseRowBinaryStream(Stri } @Override - public ClickHouseRowBinaryInputStream executeQueryClickhouseRowBinaryStream(String sql, Map additionalDBParams) throws SQLException { + public ClickHouseRowBinaryInputStream executeQueryClickhouseRowBinaryStream(String sql, + Map additionalDBParams) throws SQLException { return executeQueryClickhouseRowBinaryStream(sql, additionalDBParams, null); } @Override - public ClickHouseRowBinaryInputStream executeQueryClickhouseRowBinaryStream(String sql, Map additionalDBParams, Map additionalRequestParams) throws SQLException { + public ClickHouseRowBinaryInputStream executeQueryClickhouseRowBinaryStream(String sql, + Map additionalDBParams, Map additionalRequestParams) + throws SQLException { additionalDBParams = importAdditionalDBParameters(additionalDBParams); parseSqlStatements(sql, ClickHouseFormat.RowBinaryWithNamesAndTypes, additionalDBParams); - InputStream is = getLastInputStream( - additionalDBParams, - null, - additionalRequestParams - ); + InputStream is = getLastInputStream(additionalDBParams, null, additionalRequestParams); ClickHouseSqlStatement parsedStmt = getLastStatement(); try { if (parsedStmt.isQuery()) { currentUpdateCount = -1; // FIXME get server timezone? - currentRowBinaryResult = new ClickHouseRowBinaryInputStream(properties.isCompress() - ? new ClickHouseLZ4Stream(is) : is, getConnection().getTimeZone(), properties, true); + currentRowBinaryResult = new ClickHouseRowBinaryInputStream( + properties.isCompress() ? new ClickHouseLZ4Stream(is) : is, getConnection().getTimeZone(), + properties, true); return currentRowBinaryResult; } else { currentUpdateCount = 0; @@ -440,7 +433,7 @@ public int executeUpdate(String sql) throws SQLException { parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, additionalDBParams); try (InputStream is = getLastInputStream(additionalDBParams, null, null)) { - //noinspection StatementWithEmptyBody + // noinspection StatementWithEmptyBody } catch (IOException e) { log.error("can not close stream: %s", e.getMessage()); } @@ -450,7 +443,8 @@ public int executeUpdate(String sql) throws SQLException { @Override public boolean execute(String sql) throws SQLException { - // currentResult is stored here. InputString and currentResult will be closed on this.close() + // currentResult is stored here. InputString and currentResult will be closed on + // this.close() return executeQuery(sql) != null; } @@ -514,7 +508,7 @@ public void cancel() throws SQLException { return; } - executeQuery(String.format("KILL QUERY WHERE query_id='%s'", queryId)); + executeQuery(String.format("KILL QUERY WHERE query_id='%s'", queryId)); } @Override @@ -584,7 +578,7 @@ public int getResultSetType() throws SQLException { @Override public void addBatch(String sql) throws SQLException { - for (ClickHouseSqlStatement s : ClickHouseSqlParser.parse(sql, properties)) { + for (ClickHouseSqlStatement s : ClickHouseSqlParser.parse(sql, null)) { this.batchStmts.add(s); } } @@ -601,7 +595,7 @@ public int[] executeBatch() throws SQLException { for (int i = 0; i < len; i++) { results[i] = executeStatement(batchStmts.get(i), null, null, null); } - + clearBatch(); return results; @@ -690,10 +684,9 @@ public ClickHouseResponseSummary getResponseSummary() { return currentSummary; } - private InputStream getLastInputStream( - Map additionalDBParams, - List externalData, - Map additionalRequestParams) throws ClickHouseException { + private InputStream getLastInputStream(Map additionalDBParams, + List externalData, Map additionalRequestParams) + throws ClickHouseException { InputStream is = null; for (int i = 0, len = parsedStmts.length; i < len; i++) { // TODO skip useless queries to reduce network calls and server load @@ -711,35 +704,25 @@ private InputStream getLastInputStream( return is; } - private InputStream getInputStream( - ClickHouseSqlStatement parsedStmt, - Map additionalClickHouseDBParams, - List externalData, - Map additionalRequestParams - ) throws ClickHouseException { + private InputStream getInputStream(ClickHouseSqlStatement parsedStmt, + Map additionalClickHouseDBParams, List externalData, + Map additionalRequestParams) throws ClickHouseException { String sql = parsedStmt.getSQL(); boolean ignoreDatabase = parsedStmt.isRecognized() && !parsedStmt.isDML() - && parsedStmt.containsKeyword("DATABASE"); + && parsedStmt.containsKeyword("DATABASE"); if (parsedStmt.getStatementType() == StatementType.USE) { currentDatabase = parsedStmt.getDatabaseOrDefault(currentDatabase); } - + log.debug("Executing SQL: %s", sql); - additionalClickHouseDBParams = addQueryIdTo( - additionalClickHouseDBParams == null - ? new EnumMap(ClickHouseQueryParam.class) - : additionalClickHouseDBParams); - - URI uri = buildRequestUri( - null, - externalData, - additionalClickHouseDBParams, - additionalRequestParams, - ignoreDatabase - ); - log.debug("Request url: %s", uri); + additionalClickHouseDBParams = addQueryIdTo(additionalClickHouseDBParams == null + ? new EnumMap(ClickHouseQueryParam.class) + : additionalClickHouseDBParams); + URI uri = buildRequestUri(null, externalData, additionalClickHouseDBParams, additionalRequestParams, + ignoreDatabase); + log.debug("Request url: %s", uri); HttpEntity requestEntity; if (externalData == null || externalData.isEmpty()) { @@ -747,20 +730,19 @@ private InputStream getInputStream( } else { MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); - ContentType queryContentType = ContentType.create(ContentType.TEXT_PLAIN.getMimeType(), StandardCharsets.UTF_8); + ContentType queryContentType = ContentType.create(ContentType.TEXT_PLAIN.getMimeType(), + StandardCharsets.UTF_8); entityBuilder.addTextBody("query", sql, queryContentType); try { for (ClickHouseExternalData externalDataItem : externalData) { - // clickhouse may return 400 (bad request) when chunked encoding is used with multipart request + // clickhouse may return 400 (bad request) when chunked encoding is used with + // multipart request // so read content to byte array to avoid chunked encoding // TODO do not read stream into memory when this issue is fixed in clickhouse - entityBuilder.addBinaryBody( - externalDataItem.getName(), - Utils.toByteArray(externalDataItem.getContent()), - ContentType.APPLICATION_OCTET_STREAM, - externalDataItem.getName() - ); + entityBuilder.addBinaryBody(externalDataItem.getName(), + Utils.toByteArray(externalDataItem.getContent()), ContentType.APPLICATION_OCTET_STREAM, + externalDataItem.getName()); } } catch (IOException e) { throw new RuntimeException(e); @@ -782,7 +764,7 @@ private InputStream getInputStream( } else { httpContext.removeAttribute("is_idempotent"); } - + HttpResponse response = client.execute(post, httpContext); entity = response.getEntity(); checkForErrorAndThrow(entity, response); @@ -797,63 +779,52 @@ private InputStream getInputStream( } // retrieve response summary - if (isQueryParamSet(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, additionalClickHouseDBParams, additionalRequestParams)) { + if (isQueryParamSet(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, additionalClickHouseDBParams, + additionalRequestParams)) { Header summaryHeader = response.getFirstHeader("X-ClickHouse-Summary"); - currentSummary = summaryHeader != null ? JsonStreamUtils.readObject(summaryHeader.getValue(), ClickHouseResponseSummary.class) : null; + currentSummary = summaryHeader != null + ? JsonStreamUtils.readObject(summaryHeader.getValue(), ClickHouseResponseSummary.class) + : null; } return is; } catch (ClickHouseException e) { throw e; } catch (Exception e) { - log.info("Error during connection to %s, reporting failure to data source, message: %s", properties, e.getMessage()); + log.info("Error during connection to %s, reporting failure to data source, message: %s", properties, + e.getMessage()); EntityUtils.consumeQuietly(entity); log.info("Error sql: %s", sql); throw ClickHouseExceptionSpecifier.specify(e, properties.getHost(), properties.getPort()); } } - URI buildRequestUri( - String sql, - List externalData, - Map additionalClickHouseDBParams, - Map additionalRequestParams, - boolean ignoreDatabase - ) { + URI buildRequestUri(String sql, List externalData, + Map additionalClickHouseDBParams, Map additionalRequestParams, + boolean ignoreDatabase) { try { - List queryParams = getUrlQueryParams( - sql, - externalData, - additionalClickHouseDBParams, - additionalRequestParams, - ignoreDatabase - ); + List queryParams = getUrlQueryParams(sql, externalData, additionalClickHouseDBParams, + additionalRequestParams, ignoreDatabase); // avoid to reuse query id if (additionalClickHouseDBParams != null) { additionalClickHouseDBParams.remove(ClickHouseQueryParam.QUERY_ID); } - return new URIBuilder() - .setScheme(properties.getSsl() ? "https" : "http") - .setHost(properties.getHost()) - .setPort(properties.getPort()) - .setPath((properties.getPath() == null || properties.getPath().isEmpty() ? "/" : properties.getPath())) - .setParameters(queryParams) - .build(); + return new URIBuilder().setScheme(properties.getSsl() ? "https" : "http").setHost(properties.getHost()) + .setPort(properties.getPort()) + .setPath((properties.getPath() == null || properties.getPath().isEmpty() ? "/" + : properties.getPath())) + .setParameters(queryParams).build(); } catch (URISyntaxException e) { log.error("Mailformed URL: %s", e.getMessage()); throw new IllegalStateException("illegal configuration of db"); } } - private List getUrlQueryParams( - String sql, - List externalData, - Map additionalClickHouseDBParams, - Map additionalRequestParams, - boolean ignoreDatabase - ) { + private List getUrlQueryParams(String sql, List externalData, + Map additionalClickHouseDBParams, Map additionalRequestParams, + boolean ignoreDatabase) { List result = new ArrayList<>(); if (sql != null && !sql.isEmpty()) { @@ -915,26 +886,34 @@ private List getUrlQueryParams( return result; } - private boolean isQueryParamSet(ClickHouseQueryParam param, Map additionalClickHouseDBParams, Map additionalRequestParams) { + private boolean isQueryParamSet(ClickHouseQueryParam param, + Map additionalClickHouseDBParams, + Map additionalRequestParams) { String value = getQueryParamValue(param, additionalClickHouseDBParams, additionalRequestParams); return "true".equals(value) || "1".equals(value); } - private String getQueryParamValue(ClickHouseQueryParam param, Map additionalClickHouseDBParams, Map additionalRequestParams) { - if (additionalRequestParams != null && additionalRequestParams.containsKey(param.getKey()) && !Utils.isNullOrEmptyString(additionalRequestParams.get(param.getKey()))) { + private String getQueryParamValue(ClickHouseQueryParam param, + Map additionalClickHouseDBParams, + Map additionalRequestParams) { + if (additionalRequestParams != null && additionalRequestParams.containsKey(param.getKey()) + && !Utils.isNullOrEmptyString(additionalRequestParams.get(param.getKey()))) { return additionalRequestParams.get(param.getKey()); } - if (getRequestParams().containsKey(param.getKey()) && !Utils.isNullOrEmptyString(getRequestParams().get(param.getKey()))) { + if (getRequestParams().containsKey(param.getKey()) + && !Utils.isNullOrEmptyString(getRequestParams().get(param.getKey()))) { return getRequestParams().get(param.getKey()); } - if (additionalClickHouseDBParams != null && additionalClickHouseDBParams.containsKey(param) && !Utils.isNullOrEmptyString(additionalClickHouseDBParams.get(param))) { + if (additionalClickHouseDBParams != null && additionalClickHouseDBParams.containsKey(param) + && !Utils.isNullOrEmptyString(additionalClickHouseDBParams.get(param))) { return additionalClickHouseDBParams.get(param); } - if (getAdditionalDBParams().containsKey(param) && !Utils.isNullOrEmptyString(getAdditionalDBParams().get(param))) { + if (getAdditionalDBParams().containsKey(param) + && !Utils.isNullOrEmptyString(getAdditionalDBParams().get(param))) { return getAdditionalDBParams().get(param); } @@ -964,21 +943,20 @@ private void setStatementPropertiesToParams(Map pa params.put(ClickHouseQueryParam.MAX_RESULT_ROWS, String.valueOf(maxRows)); params.put(ClickHouseQueryParam.RESULT_OVERFLOW_MODE, "break"); } - if(isQueryTimeoutSet) { + if (isQueryTimeoutSet) { params.put(ClickHouseQueryParam.MAX_EXECUTION_TIME, String.valueOf(queryTimeout)); } } - @Override public void sendRowBinaryStream(String sql, ClickHouseStreamCallback callback) throws SQLException { sendRowBinaryStream(sql, null, callback); } @Override - public void sendRowBinaryStream(String sql, Map additionalDBParams, ClickHouseStreamCallback callback) throws SQLException { - write().withDbParams(additionalDBParams) - .send(sql, callback, ClickHouseFormat.RowBinary); + public void sendRowBinaryStream(String sql, Map additionalDBParams, + ClickHouseStreamCallback callback) throws SQLException { + write().withDbParams(additionalDBParams).send(sql, callback, ClickHouseFormat.RowBinary); } @Override @@ -987,19 +965,15 @@ public void sendNativeStream(String sql, ClickHouseStreamCallback callback) thro } @Override - public void sendNativeStream(String sql, Map additionalDBParams, ClickHouseStreamCallback callback) throws SQLException { - write().withDbParams(additionalDBParams) - .send(sql, callback, ClickHouseFormat.Native); + public void sendNativeStream(String sql, Map additionalDBParams, + ClickHouseStreamCallback callback) throws SQLException { + write().withDbParams(additionalDBParams).send(sql, callback, ClickHouseFormat.Native); } @Override - public void sendCSVStream(InputStream content, String table, Map additionalDBParams) throws SQLException { - write() - .table(table) - .withDbParams(additionalDBParams) - .data(content) - .format(ClickHouseFormat.CSV) - .send(); + public void sendCSVStream(InputStream content, String table, Map additionalDBParams) + throws SQLException { + write().table(table).withDbParams(additionalDBParams).data(content).format(ClickHouseFormat.CSV).send(); } @Override @@ -1013,13 +987,9 @@ public void sendStream(InputStream content, String table) throws SQLException { } @Override - public void sendStream(InputStream content, String table, - Map additionalDBParams) throws SQLException { - write() - .table(table) - .data(content) - .withDbParams(additionalDBParams) - .format(ClickHouseFormat.TabSeparated) + public void sendStream(InputStream content, String table, Map additionalDBParams) + throws SQLException { + write().table(table).data(content).withDbParams(additionalDBParams).format(ClickHouseFormat.TabSeparated) .send(); } @@ -1029,21 +999,21 @@ public void sendStream(HttpEntity content, String sql) throws ClickHouseExceptio } @Deprecated - public void sendStream(HttpEntity content, String sql, - Map additionalDBParams) throws ClickHouseException { + public void sendStream(HttpEntity content, String sql, Map additionalDBParams) + throws ClickHouseException { sendStream(content, sql, ClickHouseFormat.TabSeparated, additionalDBParams); } private void sendStream(HttpEntity content, String sql, ClickHouseFormat format, - Map additionalDBParams) throws ClickHouseException { + Map additionalDBParams) throws ClickHouseException { Writer writer = write().format(format).withDbParams(additionalDBParams).sql(sql); sendStream(writer, content); } @Override - public void sendStreamSQL(InputStream content, String sql, - Map additionalDBParams) throws SQLException { + public void sendStreamSQL(InputStream content, String sql, Map additionalDBParams) + throws SQLException { write().data(content).sql(sql).withDbParams(additionalDBParams).send(); } @@ -1058,12 +1028,11 @@ void sendStream(Writer writer, HttpEntity content) throws ClickHouseException { try { String sql = writer.getSql(); boolean isContentCompressed = writer.getCompression() != ClickHouseCompression.none; - URI uri = buildRequestUri( - isContentCompressed ? sql : null, null, writer.getAdditionalDBParams(), writer.getRequestParams(), false); + URI uri = buildRequestUri(isContentCompressed ? sql : null, null, writer.getAdditionalDBParams(), + writer.getRequestParams(), false); uri = followRedirects(uri); - content = applyRequestBodyCompression( - new WrappedHttpEntity(isContentCompressed ? null : sql, content)); + content = applyRequestBodyCompression(new WrappedHttpEntity(isContentCompressed ? null : sql, content)); HttpPost httpPost = new HttpPost(uri); @@ -1076,9 +1045,12 @@ void sendStream(Writer writer, HttpEntity content) throws ClickHouseException { checkForErrorAndThrow(entity, response); // retrieve response summary - if (isQueryParamSet(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, writer.getAdditionalDBParams(), writer.getRequestParams())) { + if (isQueryParamSet(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, writer.getAdditionalDBParams(), + writer.getRequestParams())) { Header summaryHeader = response.getFirstHeader("X-ClickHouse-Summary"); - currentSummary = summaryHeader != null ? JsonStreamUtils.readObject(summaryHeader.getValue(), ClickHouseResponseSummary.class) : null; + currentSummary = summaryHeader != null + ? JsonStreamUtils.readObject(summaryHeader.getValue(), ClickHouseResponseSummary.class) + : null; } } catch (ClickHouseException e) { throw e; @@ -1089,7 +1061,8 @@ void sendStream(Writer writer, HttpEntity content) throws ClickHouseException { } } - private void checkForErrorAndThrow(HttpEntity entity, HttpResponse response) throws IOException, ClickHouseException { + private void checkForErrorAndThrow(HttpEntity entity, HttpResponse response) + throws IOException, ClickHouseException { StatusLine line = response.getStatusLine(); if (line.getStatusCode() != HttpURLConnection.HTTP_OK) { InputStream messageStream = entity.getContent(); @@ -1104,9 +1077,11 @@ private void checkForErrorAndThrow(HttpEntity entity, HttpResponse response) thr } EntityUtils.consumeQuietly(entity); if (bytes.length == 0) { - throw ClickHouseExceptionSpecifier.specify(new IllegalStateException(line.toString()), properties.getHost(), properties.getPort()); + throw ClickHouseExceptionSpecifier.specify(new IllegalStateException(line.toString()), + properties.getHost(), properties.getPort()); } else { - throw ClickHouseExceptionSpecifier.specify(new String(bytes, StandardCharsets.UTF_8), properties.getHost(), properties.getPort()); + throw ClickHouseExceptionSpecifier.specify(new String(bytes, StandardCharsets.UTF_8), + properties.getHost(), properties.getPort()); } } } @@ -1128,13 +1103,15 @@ private HttpEntity applyRequestBodyCompression(final HttpEntity entity) { return entity; } - private ClickHouseResultSet createResultSet(InputStream is, int bufferSize, String db, String table, boolean usesWithTotals, - ClickHouseStatement statement, TimeZone timezone, ClickHouseProperties properties) throws IOException { - if(isResultSetScrollable) { - return new ClickHouseScrollableResultSet(is, bufferSize, db, table, usesWithTotals, statement, timezone, properties); - } else { - return new ClickHouseResultSet(is, bufferSize, db, table, usesWithTotals, statement, timezone, properties); - } + private ClickHouseResultSet createResultSet(InputStream is, int bufferSize, String db, String table, + boolean usesWithTotals, ClickHouseStatement statement, TimeZone timezone, ClickHouseProperties properties) + throws IOException { + if (isResultSetScrollable) { + return new ClickHouseScrollableResultSet(is, bufferSize, db, table, usesWithTotals, statement, timezone, + properties); + } else { + return new ClickHouseResultSet(is, bufferSize, db, table, usesWithTotals, statement, timezone, properties); + } } private Map addQueryIdTo(Map parameters) { diff --git a/clickhouse-jdbc/src/main/java9/module-info.java b/clickhouse-jdbc/src/main/java9/module-info.java index 79719b6dd..12ffff856 100644 --- a/clickhouse-jdbc/src/main/java9/module-info.java +++ b/clickhouse-jdbc/src/main/java9/module-info.java @@ -9,8 +9,6 @@ exports ru.yandex.clickhouse.settings; exports ru.yandex.clickhouse.util; - requires java.base; - requires transitive com.clickhouse.client; requires transitive com.google.gson; requires transitive org.apache.httpcomponents.httpclient; diff --git a/clickhouse-jdbc/src/main/javacc/ClickHouseSqlParser.jj b/clickhouse-jdbc/src/main/javacc/ClickHouseSqlParser.jj index 00a590e1a..af3e59663 100644 --- a/clickhouse-jdbc/src/main/javacc/ClickHouseSqlParser.jj +++ b/clickhouse-jdbc/src/main/javacc/ClickHouseSqlParser.jj @@ -25,20 +25,21 @@ options { PARSER_BEGIN(ClickHouseSqlParser) -package ru.yandex.clickhouse.jdbc.parser; +package com.clickhouse.jdbc.parser; import java.io.StringReader; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.logging.Logger; import com.clickhouse.client.logging.LoggerFactory; -import ru.yandex.clickhouse.settings.ClickHouseProperties; - public class ClickHouseSqlParser { private static final boolean DEBUG = false; @@ -46,7 +47,7 @@ public class ClickHouseSqlParser { private final List statements = new ArrayList<>(); - private ClickHouseProperties properties; + private ClickHouseConfig config; private ParseHandler handler; private boolean tokenIn(int tokenIndex, int... tokens) { @@ -70,13 +71,13 @@ public class ClickHouseSqlParser { return !(getToken(1).kind == AND && token_source.parentToken == BETWEEN); } - public static ClickHouseSqlStatement[] parse(String sql, ClickHouseProperties properties) { - return parse(sql, properties, null); + public static ClickHouseSqlStatement[] parse(String sql, ClickHouseConfig config) { + return parse(sql, config, null); } - public static ClickHouseSqlStatement[] parse(String sql, ClickHouseProperties properties, ParseHandler handler) { - if (properties == null) { - properties = new ClickHouseProperties(); + public static ClickHouseSqlStatement[] parse(String sql, ClickHouseConfig config, ParseHandler handler) { + if (config == null) { + config = new ClickHouseConfig(); } ClickHouseSqlStatement[] stmts = new ClickHouseSqlStatement[] { @@ -86,7 +87,7 @@ public class ClickHouseSqlParser { return stmts; } - ClickHouseSqlParser p = new ClickHouseSqlParser(sql, properties, handler); + ClickHouseSqlParser p = new ClickHouseSqlParser(sql, config, handler); try { stmts = p.sql(); } catch (Exception e) { @@ -100,10 +101,10 @@ public class ClickHouseSqlParser { return stmts; } - public ClickHouseSqlParser(String sql, ClickHouseProperties properties, ParseHandler handler) { + public ClickHouseSqlParser(String sql, ClickHouseConfig config, ParseHandler handler) { this(new StringReader(sql)); - this.properties = properties; + this.config = config; this.handler = handler; } @@ -111,7 +112,7 @@ public class ClickHouseSqlParser { if (token_source.isValid()) { ClickHouseSqlStatement sqlStmt = token_source.build(handler); // FIXME remove the restriction once we can hanlde insertion with format well - if (statements.size() == 0 || sqlStmt.isRecognized()) { + if (statements.isEmpty() || sqlStmt.isRecognized()) { statements.add(sqlStmt); } } else { @@ -137,11 +138,13 @@ TOKEN_MGR_DECLS: { String cluster = null; String database = null; String table = null; + String input = null; String format = null; String outfile = null; final List parameters = new ArrayList<>(); final Map positions = new HashMap<>(); + final Map settings = new LinkedHashMap<>(); public void CommonTokenAction(Token t) { if (t.kind != ClickHouseSqlParserConstants.SEMICOLON) { @@ -174,7 +177,7 @@ TOKEN_MGR_DECLS: { m.append('#').append(name); int startPos = builder.lastIndexOf(m.toString()); - int endPos = params.size() > 0 ? builder.indexOf(")", startPos) + 1 : startPos + m.length(); + int endPos = !params.isEmpty() ? builder.indexOf(")", startPos) + 1 : startPos + m.length(); builder.delete(startPos, endPos); if (handler != null) { @@ -212,10 +215,12 @@ TOKEN_MGR_DECLS: { cluster = null; database = null; table = null; + input = null; format = null; outfile = null; parameters.clear(); positions.clear(); + settings.clear(); } ClickHouseSqlStatement build(ParseHandler handler) { @@ -223,12 +228,12 @@ TOKEN_MGR_DECLS: { ClickHouseSqlStatement s = null; if (handler != null) { s = handler.handleStatement( - sqlStmt, stmtType, cluster, database, table, format, outfile, parameters, positions); + sqlStmt, stmtType, cluster, database, table, input, format, outfile, parameters, positions, settings); } if (s == null) { s = new ClickHouseSqlStatement( - sqlStmt, stmtType, cluster, database, table, format, outfile, parameters, positions); + sqlStmt, stmtType, cluster, database, table, input, format, outfile, parameters, positions, settings); } // reset variables @@ -241,12 +246,31 @@ TOKEN_MGR_DECLS: { return validTokens > 0; } - void setPosition(String keyword) { + void addPosition(Token t) { + if (t == null) { + return; + } + + addPosition(t.image, t.beginLine == 1 ? t.beginColumn - 1 : -1); + } + + void addPosition(String keyword, int startIndex) { if (keyword == null || keyword.isEmpty()) { return; } - this.positions.put(keyword, builder.lastIndexOf(keyword)); + if (startIndex < 0) { + startIndex = builder.lastIndexOf(keyword); + } + this.positions.put(keyword.toUpperCase(Locale.ROOT), startIndex); + } + + void addSetting(String key, String value) { + if (key == null || key.isEmpty()) { + return; + } + + this.settings.put(key.toLowerCase(Locale.ROOT), value); } } @@ -278,7 +302,7 @@ ClickHouseSqlStatement[] sql(): {} { { return statements.toArray(new ClickHouseSqlStatement[statements.size()]); } } -void stmts(): { Token t; } { +void stmts(): {} { LOOKAHEAD(2) stmt() | LOOKAHEAD(2) anyExprList() // in case there's anything new } @@ -333,15 +357,15 @@ void attachStmt(): { Token t; } { ( LOOKAHEAD(2) ( - t = { token_source.setPosition(t.image); } + { token_source.addPosition(token); } | - | ( t = { token_source.setPosition(t.image); }) ( + | ( { token_source.addPosition(token); }) ( ()? | ( | )? ) ) ( LOOKAHEAD(2) - t = { token_source.setPosition(t.image); } + { token_source.addPosition(token); } )? )? anyExprList() // not interested @@ -353,19 +377,19 @@ void checkStmt(): {} { // not interested } // https://clickhouse.tech/docs/en/sql-reference/statements/create/ -void createStmt(): { Token t; } { +void createStmt(): {} { ( LOOKAHEAD(2) ( - t = { token_source.setPosition(t.image); } - | ( t = { token_source.setPosition(t.image); })? ( + { token_source.addPosition(token); } + | ( { token_source.addPosition(token); })? ( ()?
| ( | )? ) | | | | ()? | | ()? ) ( LOOKAHEAD(2) - t = { token_source.setPosition(t.image); } + { token_source.addPosition(token); } )? )? anyExprList() // not interested @@ -373,7 +397,8 @@ void createStmt(): { Token t; } { // upcoming lightweight mutation - see https://github.com/ClickHouse/ClickHouse/issues/19627 void deleteStmt(): {} { - tableIdentifier(true) ( anyExprList())? + { token_source.addPosition(token); } { token_source.addPosition(token); } tableIdentifier(true) + (LOOKAHEAD({ getToken(1).kind == WHERE }) { token_source.addPosition(token); })? (anyExprList())? } // https://clickhouse.tech/docs/en/sql-reference/statements/describe-table/ @@ -383,33 +408,33 @@ void describeStmt(): {} { } // https://clickhouse.tech/docs/en/sql-reference/statements/detach/ -void detachStmt(): { Token t; } { +void detachStmt(): {} { ( LOOKAHEAD(2) ( - t = { token_source.setPosition(t.image); } + { token_source.addPosition(token); } | ()?
| | ) ( LOOKAHEAD(2) - t = { token_source.setPosition(t.image); } + { token_source.addPosition(token); } )? )? anyExprList() // not interested } // https://clickhouse.tech/docs/en/sql-reference/statements/drop/ -void dropStmt(): { Token t; } { +void dropStmt(): {} { ( LOOKAHEAD(2) ( - t = { token_source.setPosition(t.image); } + { token_source.addPosition(token); } | ()?
| | | | | ()? | | ()? ) ( LOOKAHEAD(2) - t = { token_source.setPosition(t.image); } + { token_source.addPosition(token); } )? )? anyExprList() // not interested @@ -441,17 +466,18 @@ void insertStmt(): {} { dataClause() } -void dataClause(): { Token t; } { +void dataClause(): {} { try { - LOOKAHEAD(2) anyIdentifier() (LOOKAHEAD(2) anyExprList())? - | LOOKAHEAD(2) t = { token_source.setPosition(t.image); } + LOOKAHEAD(2) { token_source.addPosition(token); } columnExprList() ( LOOKAHEAD(2) ()? columnExprList() )* - | anyExprList() // not interested + | (LOOKAHEAD(2) ( { token_source.format = ClickHouseSqlUtils.unescape(token.image); } )? + { token_source.format = token.image; })? (anyExprList())? } catch (ParseException e) { // FIXME introduce a lexical state in next release with consideration of delimiter from the context Token nextToken; @@ -546,8 +572,9 @@ void truncateStmt(): {} { } // upcoming lightweight mutation - see https://github.com/ClickHouse/ClickHouse/issues/19627 -void updateStmt(): {} { // not interested - anyExprList() +void updateStmt(): {} { + { token_source.addPosition(token); } tableIdentifier(true) + { token_source.addPosition(token); } anyExprList() } // https://clickhouse.tech/docs/en/sql-reference/statements/use/ @@ -658,16 +685,20 @@ void columnExpr(): { Token t; } { } // interested parts -void formatPart(): { Token t; } { - (LOOKAHEAD(2) t = { token_source.format = t.image; })? +void formatPart(): {} { + (LOOKAHEAD(2) { token_source.format = token.image; })? } -void outfilePart(): { Token t; } { - (LOOKAHEAD(2) t = { token_source.outfile = t.image; })? +void outfilePart(): {} { + (LOOKAHEAD(2) { token_source.outfile = token.image; })? } -void withTotalPart(): { Token t; } { - (LOOKAHEAD(2) t = { token_source.setPosition(t.image); })? +void settingsPart(): {} { + { token_source.addPosition(token); } settingExprList() +} + +void withTotalPart(): {} { + (LOOKAHEAD(2) { token_source.addPosition(token); })? } // expressions @@ -687,6 +718,7 @@ void anyExpr(): {} { void anyNestedExpr(): {} { LOOKAHEAD(2) formatPart() + | LOOKAHEAD(2) settingsPart() | LOOKAHEAD(2) withTotalPart() | LOOKAHEAD(2) outfilePart() | (LOOKAHEAD(2) )? anyColumnExpr() ( @@ -695,9 +727,9 @@ void anyNestedExpr(): {} { )* } -void anyColumnExpr(): { Token t; } { +void anyColumnExpr(): {} { // - t = { token_source.processParameter(t.image, handler); } + { token_source.processParameter(token.image, handler); } | (LOOKAHEAD(2) anyExprList())? | (LOOKAHEAD(2) anyExprList())? | (LOOKAHEAD(2) anyExprList())? @@ -711,6 +743,7 @@ Token aliasExpr(): { Token t = null; } { ( LOOKAHEAD(2) t = anyIdentifier() | LOOKAHEAD(2) formatPart() + | LOOKAHEAD(2) settingsPart() | LOOKAHEAD(2) outfilePart() | t = identifier() ) @@ -738,11 +771,11 @@ void databaseIdentifier(boolean record): { Token t; } { } void settingExprList(): {} { - settingExpr() ( settingExpr())* + settingExpr() (LOOKAHEAD(2) settingExpr())* } -void settingExpr(): {} { - identifier() literal() +void settingExpr(): { String key; } { + identifier() { key = token.image; } literal() { token_source.addSetting(key, token.image); } } // basics @@ -782,10 +815,10 @@ Token literal(): { Token t; } { { return t; } } -Token dateLiteral(): { Token t; String prefix; } { - (t = | t = ) { prefix = t.image; } +Token dateLiteral(): { Token t; StringBuilder sb = new StringBuilder(); } { + (t = | t = ) { sb.append(t.image).append(' '); } t = - { return Token.newToken(0, prefix + " " + t.image); } + { return Token.newToken(0, sb.append(t.image).toString()); } } Token numberLiteral(): { Token t = null; StringBuilder sb = new StringBuilder(); } { @@ -800,7 +833,7 @@ Token numberLiteral(): { Token t = null; StringBuilder sb = new StringBuilder(); } void operator(): {} { - ( | | | | | + ( | | | | | | | | | | | | | | ) } @@ -841,7 +874,7 @@ Token anyKeyword(): { Token t; } { | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = - | t = | t = | t = | t = | t = | t = | t = | t = + | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t =
| t = | t = | t = | t = @@ -935,6 +968,7 @@ TOKEN: { | > | > | > + |

> | > | > | > @@ -1040,6 +1074,7 @@ TOKEN: { | | | + | | | | diff --git a/clickhouse-jdbc/src/main/resources/META-INF/services/java.sql.Driver b/clickhouse-jdbc/src/main/resources/META-INF/services/java.sql.Driver index a0b809b09..1295d85a1 100644 --- a/clickhouse-jdbc/src/main/resources/META-INF/services/java.sql.Driver +++ b/clickhouse-jdbc/src/main/resources/META-INF/services/java.sql.Driver @@ -1 +1 @@ -ru.yandex.clickhouse.ClickHouseDriver +com.clickhouse.jdbc.ClickHouseDriver diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaDataTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaDataTest.java new file mode 100644 index 000000000..bf294df05 --- /dev/null +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaDataTest.java @@ -0,0 +1,21 @@ +package com.clickhouse.jdbc; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Properties; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseDatabaseMetaDataTest extends JdbcIntegrationTest { + @Test(groups = "integration") + public void testGetTypeInfo() throws SQLException { + Properties props = new Properties(); + props.setProperty("decompress", "0"); + try (ClickHouseConnection conn = newConnection(props); ResultSet rs = conn.getMetaData().getTypeInfo()) { + while (rs.next()) { + Assert.assertNotNull(rs.getString(1)); + } + } + } +} diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDriverTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDriverTest.java new file mode 100644 index 000000000..a4285c1e9 --- /dev/null +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDriverTest.java @@ -0,0 +1,28 @@ +package com.clickhouse.jdbc; + +import java.sql.SQLException; + +import com.clickhouse.client.ClickHouseProtocol; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseDriverTest extends JdbcIntegrationTest { + @Test(groups = "integration") + public void testAcceptUrl() throws SQLException { + String address = getServerAddress(ClickHouseProtocol.HTTP, true); + ClickHouseDriver driver = new ClickHouseDriver(); + Assert.assertTrue(driver.acceptsURL("jdbc:clickhouse://" + address)); + Assert.assertTrue(driver.acceptsURL("jdbc:clickhouse:http://" + address)); + Assert.assertTrue(driver.acceptsURL("jdbc:ch://" + address)); + Assert.assertTrue(driver.acceptsURL("jdbc:ch:http://" + address)); + } + + @Test(groups = "integration") + public void testConnect() throws SQLException { + String address = getServerAddress(ClickHouseProtocol.HTTP, true); + ClickHouseDriver driver = new ClickHouseDriver(); + ClickHouseConnection conn = driver.connect("jdbc:clickhouse://" + address, null); + conn.close(); + } +} diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHousePreparedStatementTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHousePreparedStatementTest.java new file mode 100644 index 000000000..e43e94791 --- /dev/null +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHousePreparedStatementTest.java @@ -0,0 +1,71 @@ +package com.clickhouse.jdbc; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Properties; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHousePreparedStatementTest extends JdbcIntegrationTest { + @Test(groups = "integration") + public void testBatchInsert() throws SQLException { + try (ClickHouseConnection conn = newConnection(new Properties()); + PreparedStatement stmt = conn.prepareStatement("insert into test_batch_insert values(?,?)")) { + conn.createStatement().execute("drop table if exists test_batch_insert;" + + "create table test_batch_insert(id Int32, name Nullable(String))engine=Memory"); + stmt.setInt(1, 1); + stmt.setString(2, "a"); + stmt.addBatch(); + stmt.setInt(1, 2); + stmt.setString(2, "b"); + stmt.addBatch(); + stmt.setInt(1, 3); + stmt.setString(2, null); + stmt.addBatch(); + int[] results = stmt.executeBatch(); + Assert.assertEquals(results, new int[] { 0, 0, 0 }); + } + } + + @Test(groups = "integration") + public void testBatchInput() throws SQLException { + try (ClickHouseConnection conn = newConnection(new Properties()); + PreparedStatement stmt = conn.prepareStatement( + "insert into test_batch_input select id, name from input('id Int32, name Nullable(String), desc Nullable(String)')")) { + conn.createStatement().execute("drop table if exists test_batch_input;" + + "create table test_batch_input(id Int32, name Nullable(String))engine=Memory"); + stmt.setInt(1, 1); + stmt.setString(2, "a"); + stmt.setString(3, "aaaaa"); + stmt.addBatch(); + stmt.setInt(1, 2); + stmt.setString(2, "b"); + stmt.setString(3, null); + stmt.addBatch(); + stmt.setInt(1, 3); + stmt.setString(2, null); + stmt.setString(3, "33333"); + stmt.addBatch(); + int[] results = stmt.executeBatch(); + Assert.assertEquals(results, new int[] { 0, 0, 0 }); + } + } + + @Test(groups = "integration") + public void testBatchQuery() throws SQLException { + try (ClickHouseConnection conn = newConnection(new Properties()); + PreparedStatement stmt = conn.prepareStatement("select * from numbers(100) where number < ?")) { + Assert.assertThrows(SQLException.class, () -> stmt.setInt(0, 5)); + Assert.assertThrows(SQLException.class, () -> stmt.setInt(2, 5)); + Assert.assertThrows(SQLException.class, () -> stmt.addBatch()); + + stmt.setInt(1, 3); + stmt.addBatch(); + stmt.setInt(1, 2); + stmt.addBatch(); + int[] results = stmt.executeBatch(); + Assert.assertEquals(results, new int[] { 0, 0 }); + } + } +} diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java new file mode 100644 index 000000000..f88b3372b --- /dev/null +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java @@ -0,0 +1,208 @@ +package com.clickhouse.jdbc; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.Locale; +import java.util.Properties; +import java.util.TimeZone; +import java.util.UUID; + +import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.data.ClickHouseDateTimeValue; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseStatementTest extends JdbcIntegrationTest { + @Test(groups = "local") + public void testLogComment() throws SQLException { + Properties props = new Properties(); + props.setProperty(ClickHouseClientOption.LOG_LEADING_COMMENT.getKey(), "true"); + try (ClickHouseConnection conn = newConnection(props)) { + ClickHouseStatement stmt = conn.createStatement(); + String uuid = UUID.randomUUID().toString(); + String sql = "-- select something " + uuid + "\nselect 12345"; + stmt.execute(sql + "; system flush logs;"); + ResultSet rs = stmt.executeQuery( + "select distinct query from system.query_log where log_comment = 'select something " + uuid + "'"); + Assert.assertTrue(rs.next()); + Assert.assertEquals(rs.getString(1), sql); + Assert.assertFalse(rs.next()); + } + } + + @Test(groups = "integration") + public void testMutation() throws SQLException { + try (ClickHouseConnection conn = newConnection(new Properties())) { + ClickHouseStatement stmt = conn.createStatement(); + stmt.execute("drop table if exists test_mutation;" + + "create table test_mutation(a String, b UInt32) engine=MergeTree() order by tuple()"); + // [delete from ]tbl a [delete ]where a.b = 1[ settings mutation_async=0] + // alter table tbl a delete where a.b = 1 + stmt.execute("-- test\nselect 1"); + stmt.execute("-- test\ndelete from test_mutation where b = 1"); + // [update] tbl a [set] a.b = 1 where a.b != 1[ settings mutation_async=0] + // alter table tbl a update a.b = 1 where a.b != 1 + conn.setClientInfo("ApplicationName", "333"); + conn.createStatement().execute("update test_mutation set b = 22 where b = 1"); + } + } + + @Test(groups = "integration") + public void testQuery() throws SQLException { + try (ClickHouseConnection conn = newConnection(new Properties())) { + ClickHouseStatement stmt = conn.createStatement(); + stmt.setMaxRows(10); + ResultSet rs = stmt.executeQuery("select * from numbers(100)"); + + try (ResultSet colRs = conn.getMetaData().getColumns(null, "system", "query_log", "")) { + while (colRs.next()) { + continue; + } + } + + while (rs.next()) { + continue; + } + } + } + + @Test(groups = "integration") + public void testTimeZone() throws SQLException { + String dateType = "DateTime32"; + String dateValue = "2020-02-11 00:23:33"; + ClickHouseDateTimeValue v = ClickHouseDateTimeValue.of(dateValue, 0); + + Properties props = new Properties(); + String[] timeZones = new String[] { "Asia/Chongqing", "America/Los_Angeles", "Europe/Moscow", "Etc/UTC", + "Europe/Berlin" }; + StringBuilder columns = new StringBuilder().append("d0 ").append(dateType); + StringBuilder constants = new StringBuilder().append(ClickHouseValues.convertToQuotedString(dateValue)); + StringBuilder currents = new StringBuilder().append("now()"); + StringBuilder parameters = new StringBuilder().append("?,?"); + int len = timeZones.length; + Calendar[] calendars = new Calendar[len + 1]; + for (int i = 0; i < len; i++) { + String timeZoneId = timeZones[i]; + columns.append(",d").append(i + 1).append(' ').append(dateType).append("('").append(timeZoneId) + .append("')"); + constants.append(',').append(ClickHouseValues.convertToQuotedString(dateValue)); + currents.append(",now()"); + parameters.append(",?"); + calendars[i] = new GregorianCalendar(TimeZone.getTimeZone(timeZoneId)); + } + len++; + try (ClickHouseConnection conn = newConnection(props); + Connection mconn = newMySqlConnection(props); + Statement mstmt = mconn.createStatement();) { + ClickHouseStatement stmt = conn.createStatement(); + stmt.execute("drop table if exists test_tz;" + "create table test_tz(no String," + columns.toString() + + ") engine=Memory;" + "insert into test_tz values('0 - Constant'," + constants.toString() + ");" + + "insert into test_tz values('1 - Current'," + currents.toString() + ");"); + + String sql = "insert into test_tz values(" + parameters.toString() + ")"; + try (PreparedStatement ps = conn.prepareStatement(sql); + PreparedStatement mps = mconn.prepareStatement(sql)) { + int index = 2; + mps.setString(1, (0 - index) + " - String"); + ps.setString(1, index++ + " - String"); + for (int i = 1; i <= len; i++) { + ps.setString(i + 1, v.asString()); + mps.setString(i + 1, v.asString()); + } + ps.addBatch(); + mps.addBatch(); + + ps.setString(1, index++ + " - LocalDateTime"); + for (int i = 1; i <= len; i++) { + ps.setObject(i + 1, v.asDateTime()); + } + ps.addBatch(); + + ps.setString(1, index++ + " - OffsetDateTime"); + for (int i = 1; i <= len; i++) { + ps.setObject(i + 1, v.asOffsetDateTime()); + } + ps.addBatch(); + + ps.setString(1, index++ + " - DateTime"); + for (int i = 1; i <= len; i++) { + if (i == 1) { + ps.setObject(i + 1, v.asDateTime()); + } else { + ps.setObject(i + 1, v.asDateTime().atZone(TimeZone.getTimeZone(timeZones[i - 2]).toZoneId()) + .toOffsetDateTime()); + } + } + ps.addBatch(); + + mps.setString(1, (0 - index) + " - BigDecimal"); + ps.setString(1, index++ + " - BigDecimal"); + for (int i = 1; i <= len; i++) { + ps.setBigDecimal(i + 1, v.asBigDecimal()); + mps.setBigDecimal(i + 1, v.asBigDecimal()); + } + ps.addBatch(); + mps.addBatch(); + + mps.setString(1, (0 - index) + " - Timestamp"); + ps.setString(1, index++ + " - Timestamp"); + for (int i = 1; i <= len; i++) { + ps.setTimestamp(i + 1, Timestamp.valueOf(v.asDateTime())); + mps.setTimestamp(i + 1, Timestamp.valueOf(v.asDateTime())); + } + ps.addBatch(); + mps.addBatch(); + + for (int j = 0; j < len; j++) { + Calendar c = calendars[j]; + mps.setString(1, (0 - index) + " - Timestamp(" + (c == null ? "" : c.getTimeZone().getID()) + ")"); + ps.setString(1, index++ + " - Timestamp(" + (c == null ? "" : c.getTimeZone().getID()) + ")"); + for (int i = 1; i <= len; i++) { + ps.setTimestamp(i + 1, Timestamp.valueOf(v.asDateTime()), c); + mps.setTimestamp(i + 1, Timestamp.valueOf(v.asDateTime()), c); + } + ps.addBatch(); + mps.addBatch(); + } + + int[] results = ps.executeBatch(); + mps.executeBatch(); + } + + try (ResultSet rs = stmt + .executeQuery("select * from test_tz order by toInt32(splitByString(' - ', no)[1])"); + ResultSet mrs = mstmt + .executeQuery("select * from test_tz order by toInt32(splitByString(' - ', no)[1])")) { + int row = 0; + while (rs.next()) { + row++; + Assert.assertTrue(mrs.next()); + + for (int i = 1; i <= len; i++) { + String msg = String.format(Locale.ROOT, "row: %d, column: %d", row, i + 1); + // Assert.assertEquals(rs.getObject(i + 1), mrs.getObject(i + 1)); + Assert.assertEquals(rs.getDate(i + 1), mrs.getDate(i + 1), msg); + Assert.assertEquals(rs.getString(i + 1), mrs.getString(i + 1), msg); + Assert.assertEquals(rs.getTimestamp(i + 1), mrs.getTimestamp(i + 1), msg); + Assert.assertEquals(rs.getTime(i + 1), mrs.getTime(i + 1), msg); + for (int j = 0; j < len; j++) { + msg = String.format(Locale.ROOT, "row: %d, column: %d, calendar: %s", row, i + 1, + calendars[j]); + Assert.assertEquals(rs.getTimestamp(i + 1, calendars[j]), + mrs.getTimestamp(i + 1, calendars[j]), msg); + Assert.assertEquals(rs.getTime(i + 1, calendars[j]), mrs.getTime(i + 1, calendars[j]), msg); + } + } + } + } + } + } +} diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/CombinedResultSetTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/CombinedResultSetTest.java new file mode 100644 index 000000000..c64ee9f99 --- /dev/null +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/CombinedResultSetTest.java @@ -0,0 +1,115 @@ +package com.clickhouse.jdbc; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collections; + +import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.client.data.ClickHouseSimpleResponse; + +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class CombinedResultSetTest { + @DataProvider(name = "multipleResultSetsProvider") + private Object[][] getMultipleResultSets() { + return new Object[][] { + { new CombinedResultSet(null, new ClickHouseResultSet("", "", + ClickHouseSimpleResponse.of(ClickHouseColumn.parse("s String"), + new Object[][] { new Object[] { "a" }, new Object[] { "b" } })), + new ClickHouseResultSet("", "", + ClickHouseSimpleResponse.of(ClickHouseColumn.parse("s String"), + new Object[][] { new Object[] { "c" }, new Object[] { "d" }, + new Object[] { "e" } }))) }, + { new CombinedResultSet(Arrays.asList(null, null, + new ClickHouseResultSet("", "", + ClickHouseSimpleResponse.of( + ClickHouseColumn.parse("s String"), new Object[][] { new Object[] { "a" } })), + null, + new ClickHouseResultSet("", "", + ClickHouseSimpleResponse.of(ClickHouseColumn.parse("s String"), + new Object[][] { new Object[] { "b" } })), + new ClickHouseResultSet("", "", + ClickHouseSimpleResponse.of(ClickHouseColumn.parse("s String"), new Object[][] { + new Object[] { "c" }, new Object[] { "d" }, new Object[] { "e" } })))) } }; + } + + @DataProvider(name = "nullOrEmptyResultSetProvider") + private Object[][] getNullOrEmptyResultSet() { + return new Object[][] { { new CombinedResultSet() }, { new CombinedResultSet((ResultSet) null) }, + { new CombinedResultSet(null, null) }, { new CombinedResultSet(null, null, null) }, + { new CombinedResultSet(Collections.emptyList()) }, + { new CombinedResultSet(Collections.singleton(null)) }, + { new CombinedResultSet(Arrays.asList(null, null)) }, + { new CombinedResultSet(Arrays.asList(null, null, null)) } }; + } + + @DataProvider(name = "singleResultSetProvider") + private Object[][] getSingleResultSet() { + return new Object[][] { + { new CombinedResultSet(new ClickHouseResultSet("", "", + ClickHouseSimpleResponse.of(ClickHouseColumn.parse("s String"), + new Object[][] { new Object[] { "a" }, new Object[] { "b" } }))) }, + { new CombinedResultSet(Collections.singleton( + new ClickHouseResultSet("", "", ClickHouseSimpleResponse.of(ClickHouseColumn.parse("s String"), + new Object[][] { new Object[] { "a" }, new Object[] { "b" } })))) } }; + } + + @Test(dataProvider = "multipleResultSetsProvider", groups = "unit") + public void testMultipleResultSets(CombinedResultSet combined) throws SQLException { + Assert.assertFalse(combined.isClosed()); + Assert.assertEquals(combined.getRow(), 0); + Assert.assertTrue(combined.next()); + Assert.assertEquals(combined.getRow(), 1); + Assert.assertEquals(combined.getString(1), "a"); + Assert.assertTrue(combined.next()); + Assert.assertEquals(combined.getRow(), 2); + Assert.assertEquals(combined.getString(1), "b"); + Assert.assertTrue(combined.next()); + Assert.assertEquals(combined.getRow(), 3); + Assert.assertEquals(combined.getString(1), "c"); + Assert.assertTrue(combined.next()); + Assert.assertEquals(combined.getRow(), 4); + Assert.assertEquals(combined.getString(1), "d"); + Assert.assertTrue(combined.next()); + Assert.assertEquals(combined.getRow(), 5); + Assert.assertEquals(combined.getString(1), "e"); + Assert.assertFalse(combined.next()); + Assert.assertFalse(combined.next()); + Assert.assertEquals(combined.getRow(), 5); + combined.close(); + Assert.assertTrue(combined.isClosed()); + } + + @Test(dataProvider = "nullOrEmptyResultSetProvider", groups = "unit") + public void testNullAndEmptyResultSet(CombinedResultSet combined) throws SQLException { + Assert.assertFalse(combined.isClosed()); + Assert.assertEquals(combined.getRow(), 0); + Assert.assertFalse(combined.next()); + Assert.assertEquals(combined.getRow(), 0); + Assert.assertFalse(combined.next()); + Assert.assertEquals(combined.getRow(), 0); + combined.close(); + Assert.assertTrue(combined.isClosed()); + Assert.assertThrows(SQLException.class, () -> combined.getString(1)); + } + + @Test(dataProvider = "singleResultSetProvider", groups = "unit") + public void testSingleResultSet(CombinedResultSet combined) throws SQLException { + Assert.assertFalse(combined.isClosed()); + Assert.assertEquals(combined.getRow(), 0); + Assert.assertTrue(combined.next()); + Assert.assertEquals(combined.getRow(), 1); + Assert.assertEquals(combined.getString(1), "a"); + Assert.assertTrue(combined.next()); + Assert.assertEquals(combined.getRow(), 2); + Assert.assertEquals(combined.getString(1), "b"); + Assert.assertFalse(combined.next()); + Assert.assertFalse(combined.next()); + Assert.assertEquals(combined.getRow(), 2); + combined.close(); + Assert.assertTrue(combined.isClosed()); + } +} diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/JdbcIntegrationTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/JdbcIntegrationTest.java new file mode 100644 index 000000000..78488f013 --- /dev/null +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/JdbcIntegrationTest.java @@ -0,0 +1,138 @@ +package com.clickhouse.jdbc; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +import com.clickhouse.client.BaseIntegrationTest; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseProtocol; + +public abstract class JdbcIntegrationTest extends BaseIntegrationTest { + private static final String CLASS_PREFIX = "ClickHouse"; + private static final String CLASS_SUFFIX = "Test"; + + protected final String dbName; + + protected String buildJdbcUrl(ClickHouseProtocol protocol, String prefix, String url) { + if (url != null && url.startsWith("jdbc:")) { + return url; + } + + StringBuilder builder = new StringBuilder(); + if (prefix == null || prefix.isEmpty()) { + builder.append("jdbc:clickhouse://"); + } else if (!prefix.startsWith("jdbc:")) { + builder.append("jdbc:").append(prefix); + } else { + builder.append(prefix); + } + + builder.append(getServerAddress(protocol)); + + if (url != null && !url.isEmpty()) { + if (url.charAt(0) != '/') { + builder.append('/'); + } + + builder.append(url); + } + + return builder.toString(); + } + + public JdbcIntegrationTest() { + String className = getClass().getSimpleName(); + if (className.startsWith(CLASS_PREFIX)) { + className = className.substring(CLASS_PREFIX.length()); + } + if (className.endsWith(CLASS_SUFFIX)) { + className = className.substring(0, className.length() - CLASS_SUFFIX.length()); + } + + this.dbName = "test_" + className.toLowerCase(); + } + + public String getServerAddress(ClickHouseProtocol protocol) { + return getServerAddress(protocol, false); + } + + public String getServerAddress(ClickHouseProtocol protocol, boolean useIPaddress) { + ClickHouseNode server = getServer(protocol); + + return new StringBuilder().append(useIPaddress ? getIpAddress(server) : server.getHost()).append(':') + .append(server.getPort()).toString(); + } + + public String getServerAddress(ClickHouseProtocol protocol, String customHostOrIp) { + ClickHouseNode server = getServer(protocol); + return new StringBuilder() + .append(customHostOrIp == null || customHostOrIp.isEmpty() ? server.getHost() : customHostOrIp) + .append(':').append(server.getPort()).toString(); + } + + public ClickHouseDataSource newDataSource() { + return newDataSource(null, new Properties()); + } + + public ClickHouseDataSource newDataSource(Properties properties) { + return newDataSource(null, properties); + } + + public ClickHouseDataSource newDataSource(String url) { + return newDataSource(url, new Properties()); + } + + public ClickHouseDataSource newDataSource(String url, Properties properties) { + return new ClickHouseDataSource(buildJdbcUrl(ClickHouseProtocol.HTTP, null, url), properties); + } + + public ClickHouseConnection newConnection() throws SQLException { + return newConnection(null); + } + + public ClickHouseConnection newConnection(Properties properties) throws SQLException { + try (ClickHouseConnection conn = newDataSource(properties).getConnection(); + ClickHouseStatement stmt = conn.createStatement();) { + stmt.execute("CREATE DATABASE IF NOT EXISTS " + dbName); + } + + return newDataSource(dbName, properties == null ? new Properties() : properties).getConnection(); + } + + public Connection newMySqlConnection(Properties properties) throws SQLException { + if (properties == null) { + properties = new Properties(); + } + + if (!properties.containsKey("user")) { + properties.setProperty("user", "default"); + } + if (!properties.containsKey("password")) { + properties.setProperty("password", ""); + } + + Connection conn = DriverManager.getConnection(buildJdbcUrl(ClickHouseProtocol.MYSQL, "jdbc:mysql://", dbName), + properties); + + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE DATABASE IF NOT EXISTS " + dbName); + } + + return conn; + } + + public void closeConnection(Connection conn) throws SQLException { + if (conn == null) { + return; + } + + try (Statement stmt = conn.createStatement()) { + stmt.execute("DROP DATABASE IF EXISTS " + dbName); + } finally { + conn.close(); + } + } +} diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/JdbcParameterizedQueryTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/JdbcParameterizedQueryTest.java new file mode 100644 index 000000000..40e20308e --- /dev/null +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/JdbcParameterizedQueryTest.java @@ -0,0 +1,44 @@ +package com.clickhouse.jdbc; + +import java.util.Arrays; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class JdbcParameterizedQueryTest { + @Test(groups = "unit") + public void testParseBlankQueries() { + Assert.assertThrows(IllegalArgumentException.class, () -> JdbcParameterizedQuery.of(null)); + Assert.assertThrows(IllegalArgumentException.class, () -> JdbcParameterizedQuery.of("")); + Assert.assertThrows(IllegalArgumentException.class, () -> JdbcParameterizedQuery.of(" \n\t\r")); + } + + @Test(groups = "unit") + public void testParseQueriesWithNamedParameter() { + String sql = "select :no, :name(String)"; + JdbcParameterizedQuery q = JdbcParameterizedQuery.of(sql); + Assert.assertEquals(q.getOriginalQuery(), sql); + Assert.assertEquals(q.hasParameter(), false); + } + + @Test(groups = "unit") + public void testParseJdbcQueries() { + String sql = "select ?(number % 2 == 0 ? 1 : 0) from numbers(100) where number > ?"; + JdbcParameterizedQuery q = JdbcParameterizedQuery.of(sql); + Assert.assertEquals(q.getOriginalQuery(), sql); + Assert.assertEquals(q.hasParameter(), true); + Assert.assertEquals(q.getNamedParameters(), Arrays.asList("0", "1")); + Assert.assertEquals(q.apply("sum", "1"), + "select sum(number % 2 == 0 ? 1 : 0) from numbers(100) where number > 1"); + + Assert.assertEquals(JdbcParameterizedQuery.of("select '; select 2' as ?").hasParameter(), true); + Assert.assertThrows(IllegalArgumentException.class, () -> JdbcParameterizedQuery.of("select 1; select 2")); + + sql = "select 1 ? 'a' : 'b', 2 ? (select 1) : 2, ?"; + q = JdbcParameterizedQuery.of(sql); + Assert.assertEquals(q.getOriginalQuery(), sql); + Assert.assertEquals(q.hasParameter(), true); + Assert.assertEquals(q.getNamedParameters(), Arrays.asList("0")); + Assert.assertEquals(q.apply("3"), "select 1 ? 'a' : 'b', 2 ? (select 1) : 2, 3"); + } +} diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/JdbcParseHandlerTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/JdbcParseHandlerTest.java new file mode 100644 index 000000000..8cad2cfdf --- /dev/null +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/JdbcParseHandlerTest.java @@ -0,0 +1,262 @@ +package com.clickhouse.jdbc; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import com.clickhouse.jdbc.parser.ClickHouseSqlParser; +import com.clickhouse.jdbc.parser.ClickHouseSqlStatement; +import com.clickhouse.jdbc.parser.StatementType; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class JdbcParseHandlerTest { + @Test(groups = "unit") + public void testParseDeleteStatement() { + Assert.assertEquals(ClickHouseSqlParser.parse("delete from tbl", null, JdbcParseHandler.INSTANCE)[0].getSQL(), + "TRUNCATE TABLE tbl"); + Assert.assertEquals( + ClickHouseSqlParser.parse("delete from tbl where 1", null, JdbcParseHandler.INSTANCE)[0].getSQL(), + "ALTER TABLE `tbl` DELETE where 1 SETTINGS mutations_sync=1"); + Assert.assertEquals( + ClickHouseSqlParser.parse("delete from tbl where 1 and 1 settings a=1 format CSV", null, + JdbcParseHandler.INSTANCE)[0].getSQL(), + "ALTER TABLE `tbl` DELETE where 1 and 1 SETTINGS mutations_sync=1, a=1 format CSV"); + Assert.assertEquals( + ClickHouseSqlParser.parse("delete from tbl where 1 and 1 settings mutations_sync=0 format CSV", null, + JdbcParseHandler.INSTANCE)[0].getSQL(), + "ALTER TABLE `tbl` DELETE where 1 and 1 settings mutations_sync=0 format CSV"); + } + + @Test(groups = "unit") + public void testParseInsertStatement() { + } + + @Test(groups = "unit") + public void testParseUpdateStatement() { + Assert.assertEquals( + ClickHouseSqlParser.parse("update tbl set a=1", null, JdbcParseHandler.INSTANCE)[0].getSQL(), + "ALTER TABLE `tbl` UPDATE a=1 SETTINGS mutations_sync=1"); + Assert.assertEquals( + ClickHouseSqlParser.parse("update tbl set a=1,b=2 where 1", null, JdbcParseHandler.INSTANCE)[0] + .getSQL(), + "ALTER TABLE `tbl` UPDATE a=1,b=2 where 1 SETTINGS mutations_sync=1"); + Assert.assertEquals( + ClickHouseSqlParser.parse("update tbl set x=1, y = 2 where 1 and 1 settings a=1 format CSV", null, + JdbcParseHandler.INSTANCE)[0].getSQL(), + "ALTER TABLE `tbl` UPDATE x=1, y = 2 where 1 and 1 SETTINGS mutations_sync=1, a=1 format CSV"); + Assert.assertEquals( + ClickHouseSqlParser.parse("update tbl set y = 2 where 1 and 1 settings mutations_sync=0 format CSV", + null, JdbcParseHandler.INSTANCE)[0].getSQL(), + "ALTER TABLE `tbl` UPDATE y = 2 where 1 and 1 settings mutations_sync=0 format CSV"); + } + + @Test(groups = "unit") + public void testDeleteStatementWithoutWhereClause() { + Assert.assertEquals(JdbcParseHandler.INSTANCE.handleStatement("delete from `a\\`' a` . tbl", + StatementType.DELETE, null, "a\\`' a", "tbl", null, null, null, null, new HashMap() { + { + put("DELETE", 0); + put("FROM", 8); + } + }, null), new ClickHouseSqlStatement("TRUNCATE TABLE `a\\`' a` . tbl", StatementType.DELETE, null, + "a\\`' a", "tbl", null, null, null, null, null, null)); + Assert.assertEquals(JdbcParseHandler.INSTANCE.handleStatement("delete from `table\\`'1`", StatementType.DELETE, + null, null, "table1", null, null, null, null, new HashMap() { + { + put("DELETE", 0); + put("FROM", 7); + } + }, null), new ClickHouseSqlStatement("TRUNCATE TABLE `table\\`'1`", StatementType.DELETE, null, null, + "table1", null, null, null, null, null, null), + null); + } + + @Test(groups = "unit") + public void testDeleteStatementWithWhereClause() { + Map positions = new HashMap<>(); + positions.put("DELETE", 0); + positions.put("FROM", 7); + positions.put("WHERE", 28); + Assert.assertEquals( + JdbcParseHandler.INSTANCE.handleStatement("delete from `a\\`' a` . tbl where a = b", + StatementType.DELETE, null, "a\\`' a", "tbl", null, null, null, null, positions, null), + new ClickHouseSqlStatement("ALTER TABLE `a\\`' a`.`tbl` DELETE where a = b SETTINGS mutations_sync=1", + StatementType.DELETE, null, "a\\`' a", "tbl", null, null, null, null, null, null)); + positions.put("DELETE", 0); + positions.put("FROM", 8); + positions.put("WHERE", 26); + Assert.assertEquals( + JdbcParseHandler.INSTANCE.handleStatement("delete from `table\\`'1` where 1", StatementType.DELETE, + null, null, "table\\`'1", null, null, null, null, positions, null), + new ClickHouseSqlStatement("ALTER TABLE `table\\`'1` DELETE where 1 SETTINGS mutations_sync=1", + StatementType.DELETE, null, null, "table\\`'1", null, null, null, null, null, null)); + } + + @Test(groups = "unit") + public void testDeleteStatementWithSettings() { + String sql1 = "delete from tbl settings a=1"; + Map positions = new HashMap() { + { + put("DELETE", 0); + put("FROM", sql1.indexOf("from")); + put("SETTINGS", sql1.indexOf("settings")); + } + }; + Map settings = Collections.singletonMap("a", "1"); + Assert.assertEquals( + JdbcParseHandler.INSTANCE.handleStatement(sql1, StatementType.DELETE, null, null, "tbl", null, null, + null, null, positions, settings), + new ClickHouseSqlStatement("TRUNCATE TABLE tbl settings a=1", StatementType.DELETE, null, null, "tbl", + null, null, null, null, null, settings)); + + String sql2 = "delete from tbl where a != 1 and b != 2 settings a=1,b='a'"; + positions = new HashMap() { + { + put("DELETE", 0); + put("FROM", sql2.indexOf("from")); + put("WHERE", sql2.indexOf("where")); + put("SETTINGS", sql2.indexOf("settings")); + } + }; + settings = new HashMap() { + { + put("a", "1"); + put("b", "'a'"); + } + }; + Assert.assertEquals( + JdbcParseHandler.INSTANCE.handleStatement(sql2, StatementType.DELETE, null, null, "tbl", null, null, + null, null, positions, settings), + new ClickHouseSqlStatement( + "ALTER TABLE `tbl` DELETE where a != 1 and b != 2 SETTINGS mutations_sync=1, a=1,b='a'", + StatementType.DELETE, null, null, "tbl", null, null, null, null, null, settings)); + + String sql3 = "delete from tbl where a != 1 and b != 2 settings a=1,mutations_sync=2,b='a'"; + positions = new HashMap() { + { + put("DELETE", 0); + put("FROM", sql3.indexOf("from")); + put("WHERE", sql3.indexOf("where")); + put("SETTINGS", sql3.indexOf("settings")); + } + }; + settings = new HashMap() { + { + put("a", "1"); + put("mutations_sync", "2"); + put("b", "'a'"); + } + }; + Assert.assertEquals( + JdbcParseHandler.INSTANCE.handleStatement(sql3, StatementType.DELETE, null, null, "tbl", null, null, + null, null, positions, settings), + new ClickHouseSqlStatement( + "ALTER TABLE `tbl` DELETE where a != 1 and b != 2 settings a=1,mutations_sync=2,b='a'", + StatementType.DELETE, null, null, "tbl", null, null, null, null, null, settings)); + } + + @Test(groups = "unit") + public void testUpdateStatementWithoutWhereClause() { + Assert.assertEquals(JdbcParseHandler.INSTANCE.handleStatement("update `a\\`' a` . tbl set a=1", + StatementType.UPDATE, null, "a\\`' a", "tbl", null, null, null, null, new HashMap() { + { + put("UPDATE", 0); + put("SET", 23); + } + }, null), new ClickHouseSqlStatement("ALTER TABLE `a\\`' a`.`tbl` UPDATE a=1 SETTINGS mutations_sync=1", + StatementType.UPDATE, null, "a\\`' a", "tbl", null, null, null, null, null, null)); + Assert.assertEquals(JdbcParseHandler.INSTANCE.handleStatement("update `table\\`'1` set a=1", + StatementType.UPDATE, null, null, "table1", null, null, null, null, new HashMap() { + { + put("UPDATE", 0); + put("SET", 20); + } + }, null), new ClickHouseSqlStatement("ALTER TABLE `table1` UPDATE a=1 SETTINGS mutations_sync=1", + StatementType.UPDATE, null, null, "table1", null, null, null, null, null, null)); + } + + @Test(groups = "unit") + public void testUpdateStatementWithWhereClause() { + Map positions = new HashMap<>(); + positions.put("UPDATE", 0); + positions.put("SET", 23); + Assert.assertEquals( + JdbcParseHandler.INSTANCE.handleStatement("Update `a\\`' a` . tbl set a = 2 where a = b", + StatementType.UPDATE, null, "a\\`' a", "tbl", null, null, null, null, positions, null), + new ClickHouseSqlStatement( + "ALTER TABLE `a\\`' a`.`tbl` UPDATE a = 2 where a = b SETTINGS mutations_sync=1", + StatementType.UPDATE, null, "a\\`' a", "tbl", null, null, null, null, null, null)); + positions.put("UPDATE", 0); + positions.put("SET", 19); + Assert.assertEquals( + JdbcParseHandler.INSTANCE.handleStatement("update `table\\`'1` set a = b where 1", StatementType.UPDATE, + null, null, "table\\`'1", null, null, null, null, positions, null), + new ClickHouseSqlStatement("ALTER TABLE `table\\`'1` UPDATE a = b where 1 SETTINGS mutations_sync=1", + StatementType.UPDATE, null, null, "table\\`'1", null, null, null, null, null, null)); + } + + @Test(groups = "unit") + public void testUpdateStatementWithSettings() { + String sql1 = "update tbl set x=1 settings a=1"; + Map positions = new HashMap() { + { + put("UPDATE", 0); + put("SET", sql1.indexOf("set")); + put("SETTINGS", sql1.indexOf("settings")); + } + }; + Map settings = Collections.singletonMap("a", "1"); + Assert.assertEquals( + JdbcParseHandler.INSTANCE.handleStatement(sql1, StatementType.UPDATE, null, null, "tbl", null, null, + null, null, positions, settings), + new ClickHouseSqlStatement("ALTER TABLE `tbl` UPDATE x=1 SETTINGS mutations_sync=1, a=1", + StatementType.UPDATE, null, null, "tbl", null, null, null, null, null, settings)); + + String sql2 = "update tbl set x=1, y=2 where a != 1 and b != 2 settings a=1,b='a'"; + positions = new HashMap() { + { + put("UPDATE", 0); + put("SET", sql1.indexOf("set")); + put("WHERE", sql2.indexOf("where")); + put("SETTINGS", sql2.indexOf("settings")); + } + }; + settings = new HashMap() { + { + put("a", "1"); + put("b", "'a'"); + } + }; + Assert.assertEquals( + JdbcParseHandler.INSTANCE.handleStatement(sql2, StatementType.UPDATE, null, null, "tbl", null, null, + null, null, positions, settings), + new ClickHouseSqlStatement( + "ALTER TABLE `tbl` UPDATE x=1, y=2 where a != 1 and b != 2 SETTINGS mutations_sync=1, a=1,b='a'", + StatementType.UPDATE, null, null, "tbl", null, null, null, null, null, settings)); + + String sql3 = "update tbl set x=1,y=2 where a != 1 and b != 2 settings a=1,mutations_sync=2,b='a'"; + positions = new HashMap() { + { + put("UPDATE", 0); + put("SET", sql1.indexOf("set")); + put("WHERE", sql3.indexOf("where")); + put("SETTINGS", sql3.indexOf("settings")); + } + }; + settings = new HashMap() { + { + put("a", "1"); + put("mutations_sync", "2"); + put("b", "'a'"); + } + }; + Assert.assertEquals( + JdbcParseHandler.INSTANCE.handleStatement(sql3, StatementType.UPDATE, null, null, "tbl", null, null, + null, null, positions, settings), + new ClickHouseSqlStatement( + "ALTER TABLE `tbl` UPDATE x=1,y=2 where a != 1 and b != 2 settings a=1,mutations_sync=2,b='a'", + StatementType.UPDATE, null, null, "tbl", null, null, null, null, null, settings)); + } +} diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImplTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImplTest.java new file mode 100644 index 000000000..527cce12f --- /dev/null +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImplTest.java @@ -0,0 +1,204 @@ +package com.clickhouse.jdbc.internal; + +import java.sql.SQLException; +import java.sql.Savepoint; + +import com.clickhouse.jdbc.ClickHouseConnection; +import com.clickhouse.jdbc.ClickHouseStatement; +import com.clickhouse.jdbc.JdbcIntegrationTest; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseConnectionImplTest extends JdbcIntegrationTest { + @Test(groups = "integration") + public void testManualCommit() throws SQLException { + try (ClickHouseConnectionImpl conn = (ClickHouseConnectionImpl) newConnection()) { + Assert.assertEquals(conn.getAutoCommit(), true); + Assert.assertNull(conn.getTransaction(), "Should NOT have any transaction"); + conn.setAutoCommit(false); + Assert.assertEquals(conn.getAutoCommit(), false); + FakeTransaction tx = conn.getTransaction(); + Assert.assertNotNull(tx, "Should have transaction"); + Assert.assertEquals(tx.getQueries().size(), 0); + Assert.assertEquals(tx.getSavepoints().size(), 0); + try (ClickHouseStatement stmt = conn.createStatement()) { + stmt.execute("select 1; select 2"); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 0); + + Savepoint s = conn.setSavepoint("test"); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 1); + conn.releaseSavepoint(s); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 0); + + s = conn.setSavepoint("test"); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 1); + stmt.execute("select 3"); + Assert.assertEquals(tx.getQueries().size(), 3); + Assert.assertEquals(tx.getSavepoints().size(), 1); + conn.releaseSavepoint(s); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 0); + + s = conn.setSavepoint("test"); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 1); + stmt.execute("select 3; select 4"); + Assert.assertEquals(tx.getQueries().size(), 4); + Assert.assertEquals(tx.getSavepoints().size(), 1); + conn.setSavepoint(); + Assert.assertEquals(tx.getQueries().size(), 4); + Assert.assertEquals(tx.getSavepoints().size(), 2); + stmt.execute("select 5"); + Assert.assertEquals(tx.getQueries().size(), 5); + Assert.assertEquals(tx.getSavepoints().size(), 2); + conn.releaseSavepoint(s); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 0); + + conn.setSavepoint(); + conn.setSavepoint("test"); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 2); + } + + try (ClickHouseStatement stmt = conn.createStatement()) { + stmt.execute("select 6"); + Assert.assertEquals(tx.getQueries().size(), 3); + Assert.assertEquals(tx.getSavepoints().size(), 2); + } + conn.commit(); + FakeTransaction newTx = conn.getTransaction(); + Assert.assertNotEquals(newTx, tx); + Assert.assertNotNull(tx, "Should have transaction"); + Assert.assertEquals(tx.getQueries().size(), 0); + Assert.assertEquals(tx.getSavepoints().size(), 0); + Assert.assertNotNull(newTx, "Should have transaction"); + Assert.assertEquals(newTx.getQueries().size(), 0); + Assert.assertEquals(newTx.getSavepoints().size(), 0); + tx = newTx; + + try (ClickHouseStatement stmt = conn.createStatement()) { + Savepoint s = conn.setSavepoint(); + stmt.execute("select 7; select 8"); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 1); + } + conn.commit(); + newTx = conn.getTransaction(); + Assert.assertNotEquals(newTx, tx); + Assert.assertNotNull(tx, "Should have transaction"); + Assert.assertEquals(tx.getQueries().size(), 0); + Assert.assertEquals(tx.getSavepoints().size(), 0); + Assert.assertNotNull(newTx, "Should have transaction"); + Assert.assertEquals(newTx.getQueries().size(), 0); + Assert.assertEquals(newTx.getSavepoints().size(), 0); + } + } + + @Test(groups = "integration") + public void testManualRollback() throws SQLException { + try (ClickHouseConnectionImpl conn = (ClickHouseConnectionImpl) newConnection()) { + Assert.assertEquals(conn.getAutoCommit(), true); + Assert.assertNull(conn.getTransaction(), "Should NOT have any transaction"); + conn.setAutoCommit(false); + Assert.assertEquals(conn.getAutoCommit(), false); + FakeTransaction tx = conn.getTransaction(); + Assert.assertNotNull(tx, "Should have transaction"); + Assert.assertEquals(tx.getQueries().size(), 0); + Assert.assertEquals(tx.getSavepoints().size(), 0); + try (ClickHouseStatement stmt = conn.createStatement()) { + stmt.execute("select 1; select 2"); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 0); + + Savepoint s = conn.setSavepoint("test"); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 1); + conn.rollback(s); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 0); + + s = conn.setSavepoint("test"); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 1); + stmt.execute("select 3"); + Assert.assertEquals(tx.getQueries().size(), 3); + Assert.assertEquals(tx.getSavepoints().size(), 1); + conn.rollback(s); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 0); + + s = conn.setSavepoint("test"); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 1); + stmt.execute("select 3; select 4"); + Assert.assertEquals(tx.getQueries().size(), 4); + Assert.assertEquals(tx.getSavepoints().size(), 1); + conn.setSavepoint(); + Assert.assertEquals(tx.getQueries().size(), 4); + Assert.assertEquals(tx.getSavepoints().size(), 2); + stmt.execute("select 5"); + Assert.assertEquals(tx.getQueries().size(), 5); + Assert.assertEquals(tx.getSavepoints().size(), 2); + conn.rollback(s); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 0); + + conn.setSavepoint(); + conn.setSavepoint("test"); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 2); + } + + try (ClickHouseStatement stmt = conn.createStatement()) { + stmt.execute("select 6"); + Assert.assertEquals(tx.getQueries().size(), 3); + Assert.assertEquals(tx.getSavepoints().size(), 2); + } + conn.rollback(); + FakeTransaction newTx = conn.getTransaction(); + Assert.assertNotEquals(newTx, tx); + Assert.assertNotNull(tx, "Should have transaction"); + Assert.assertEquals(tx.getQueries().size(), 0); + Assert.assertEquals(tx.getSavepoints().size(), 0); + Assert.assertNotNull(newTx, "Should have transaction"); + Assert.assertEquals(newTx.getQueries().size(), 0); + Assert.assertEquals(newTx.getSavepoints().size(), 0); + tx = newTx; + + try (ClickHouseStatement stmt = conn.createStatement()) { + Savepoint s = conn.setSavepoint(); + stmt.execute("select 7; select 8"); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints().size(), 1); + } + conn.rollback(); + newTx = conn.getTransaction(); + Assert.assertNotEquals(newTx, tx); + Assert.assertNotNull(tx, "Should have transaction"); + Assert.assertEquals(tx.getQueries().size(), 0); + Assert.assertEquals(tx.getSavepoints().size(), 0); + Assert.assertNotNull(newTx, "Should have transaction"); + Assert.assertEquals(newTx.getQueries().size(), 0); + Assert.assertEquals(newTx.getSavepoints().size(), 0); + } + } + + @Test(groups = "integration") + public void testSwitchAutoCommit() throws SQLException { + try (ClickHouseConnection conn = newConnection()) { + Assert.assertEquals(conn.getAutoCommit(), true); + conn.setAutoCommit(false); + Assert.assertEquals(conn.getAutoCommit(), false); + conn.setAutoCommit(true); + Assert.assertEquals(conn.getAutoCommit(), true); + conn.setAutoCommit(false); + Assert.assertEquals(conn.getAutoCommit(), false); + } + } +} diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParserTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParserTest.java new file mode 100644 index 000000000..6a5a0dae0 --- /dev/null +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParserTest.java @@ -0,0 +1,168 @@ +package com.clickhouse.jdbc.internal; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Properties; + +import com.clickhouse.client.ClickHouseCredentials; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.config.ClickHouseDefaults; +import com.clickhouse.jdbc.internal.ClickHouseJdbcUrlParser.ConnectionInfo; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseJdbcUrlParserTest { + @Test(groups = "unit") + public void testRemoveCredentialsFromQuery() { + Assert.assertEquals(ClickHouseJdbcUrlParser.removeCredentialsFromQuery(null), null); + Assert.assertEquals(ClickHouseJdbcUrlParser.removeCredentialsFromQuery(""), null); + Assert.assertEquals(ClickHouseJdbcUrlParser.removeCredentialsFromQuery(" "), null); + Assert.assertEquals(ClickHouseJdbcUrlParser.removeCredentialsFromQuery("&"), null); + Assert.assertEquals(ClickHouseJdbcUrlParser.removeCredentialsFromQuery(" & "), null); + Assert.assertEquals(ClickHouseJdbcUrlParser.removeCredentialsFromQuery("a=1&b=2"), "a=1&b=2"); + Assert.assertEquals(ClickHouseJdbcUrlParser.removeCredentialsFromQuery("user=a"), null); + Assert.assertEquals(ClickHouseJdbcUrlParser.removeCredentialsFromQuery("password=a%20b"), null); + Assert.assertEquals(ClickHouseJdbcUrlParser.removeCredentialsFromQuery("user=default&password=a%20b"), null); + Assert.assertEquals(ClickHouseJdbcUrlParser.removeCredentialsFromQuery("user=default&a=1&password=a%20b"), + "a=1"); + } + + @Test(groups = "unit") + public void testParseInvalidUri() { + Assert.assertThrows(IllegalArgumentException.class, () -> ClickHouseJdbcUrlParser.parse(null, null)); + Assert.assertThrows(IllegalArgumentException.class, () -> ClickHouseJdbcUrlParser.parse("", null)); + Assert.assertThrows(URISyntaxException.class, () -> ClickHouseJdbcUrlParser.parse("some_invalid_uri", null)); + Assert.assertThrows(URISyntaxException.class, () -> ClickHouseJdbcUrlParser.parse("jdbc:clickhouse:.", null)); + Assert.assertThrows(URISyntaxException.class, () -> ClickHouseJdbcUrlParser.parse("jdbc:clickhouse://", null)); + Assert.assertThrows(IllegalArgumentException.class, + () -> ClickHouseJdbcUrlParser.parse("jdbc:clickhouse:///db", null)); + Assert.assertThrows(URISyntaxException.class, + () -> ClickHouseJdbcUrlParser.parse("jdbc:clickhouse://server/ ", null)); + Assert.assertThrows(URISyntaxException.class, + () -> ClickHouseJdbcUrlParser.parse("clickhouse://a:b:c@aaa", null)); + Assert.assertThrows(URISyntaxException.class, + () -> ClickHouseJdbcUrlParser.parse("clickhouse://::1:1234/a", null)); + } + + @Test(groups = "unit") + public void testParseIpv6() throws URISyntaxException { + ConnectionInfo info = ClickHouseJdbcUrlParser.parse("jdbc:clickhouse://[::1]:1234", null); + Assert.assertEquals(info.getUri(), new URI("jdbc:clickhouse:http://[::1]:1234/default")); + Assert.assertEquals(info.getServer(), + ClickHouseNode.builder().host("[::1]").port(ClickHouseProtocol.HTTP, 1234) + .database((String) ClickHouseDefaults.DATABASE.getEffectiveDefaultValue()) + .credentials(ClickHouseCredentials.fromUserAndPassword( + (String) ClickHouseDefaults.USER.getEffectiveDefaultValue(), + (String) ClickHouseDefaults.PASSWORD.getEffectiveDefaultValue())) + .build()); + } + + @Test(groups = "unit") + public void testParseAbbrevation() throws URISyntaxException { + ConnectionInfo info = ClickHouseJdbcUrlParser.parse("jdbc:ch://localhost", null); + Assert.assertEquals(info.getUri(), new URI("jdbc:clickhouse:http://localhost:8123/default")); + Assert.assertEquals(info.getServer(), + ClickHouseNode.builder().host("localhost").port(ClickHouseProtocol.HTTP) + .database((String) ClickHouseDefaults.DATABASE.getEffectiveDefaultValue()) + .credentials(ClickHouseCredentials.fromUserAndPassword( + (String) ClickHouseDefaults.USER.getEffectiveDefaultValue(), + (String) ClickHouseDefaults.PASSWORD.getEffectiveDefaultValue())) + .build()); + + info = ClickHouseJdbcUrlParser.parse("jdbc:ch:grpc://localhost", null); + Assert.assertEquals(info.getUri(), new URI("jdbc:clickhouse:grpc://localhost:9100/default")); + Assert.assertEquals(info.getServer(), + ClickHouseNode.builder().host("localhost").port(ClickHouseProtocol.GRPC) + .database((String) ClickHouseDefaults.DATABASE.getEffectiveDefaultValue()) + .credentials(ClickHouseCredentials.fromUserAndPassword( + (String) ClickHouseDefaults.USER.getEffectiveDefaultValue(), + (String) ClickHouseDefaults.PASSWORD.getEffectiveDefaultValue())) + .build()); + + info = ClickHouseJdbcUrlParser.parse("jdbc:ch:https://:letmein@[::1]:3218/db1?user=aaa", null); + Assert.assertEquals(info.getUri(), new URI("jdbc:clickhouse:http://[::1]:3218/db1")); + Assert.assertEquals(info.getServer(), ClickHouseNode.builder().host("[::1]").port(ClickHouseProtocol.HTTP, 3218) + .database("db1").credentials(ClickHouseCredentials.fromUserAndPassword("aaa", "letmein")).build()); + Assert.assertEquals(info.getProperties().getProperty("user"), "aaa"); + } + + @Test(groups = "unit") + public void testParse() throws URISyntaxException { + ConnectionInfo info = ClickHouseJdbcUrlParser.parse("jdbc:ch://localhost", null); + Assert.assertEquals(info.getUri(), new URI("jdbc:clickhouse:http://localhost:8123/default")); + Assert.assertEquals(info.getServer(), + ClickHouseNode.builder().host("localhost").port(ClickHouseProtocol.HTTP) + .database((String) ClickHouseDefaults.DATABASE.getEffectiveDefaultValue()) + .credentials(ClickHouseCredentials.fromUserAndPassword( + (String) ClickHouseDefaults.USER.getEffectiveDefaultValue(), + (String) ClickHouseDefaults.PASSWORD.getEffectiveDefaultValue())) + .build()); + + info = ClickHouseJdbcUrlParser.parse("jdbc:ch:grpc://localhost", null); + Assert.assertEquals(info.getUri(), new URI("jdbc:clickhouse:grpc://localhost:9100/default")); + Assert.assertEquals(info.getServer(), + ClickHouseNode.builder().host("localhost").port(ClickHouseProtocol.GRPC) + .database((String) ClickHouseDefaults.DATABASE.getEffectiveDefaultValue()) + .credentials(ClickHouseCredentials.fromUserAndPassword( + (String) ClickHouseDefaults.USER.getEffectiveDefaultValue(), + (String) ClickHouseDefaults.PASSWORD.getEffectiveDefaultValue())) + .build()); + + info = ClickHouseJdbcUrlParser.parse("jdbc:ch:https://:letmein@127.0.0.1:3218/db1", null); + Assert.assertEquals(info.getUri(), new URI("jdbc:clickhouse:http://127.0.0.1:3218/db1")); + Assert.assertEquals(info.getServer(), ClickHouseNode.builder().host("127.0.0.1") + .port(ClickHouseProtocol.HTTP, 3218).database("db1") + .credentials(ClickHouseCredentials + .fromUserAndPassword((String) ClickHouseDefaults.USER.getEffectiveDefaultValue(), "letmein")) + .build()); + } + + @Test(groups = "unit") + public void testParseWithProperties() throws URISyntaxException { + ConnectionInfo info = ClickHouseJdbcUrlParser.parse("jdbc:clickhouse://localhost/", null); + Assert.assertEquals(info.getUri(), new URI("jdbc:clickhouse:http://localhost:8123/default")); + Assert.assertEquals(info.getServer(), + ClickHouseNode.builder().host("localhost").port(ClickHouseProtocol.HTTP) + .database((String) ClickHouseDefaults.DATABASE.getEffectiveDefaultValue()) + .credentials(ClickHouseCredentials.fromUserAndPassword( + (String) ClickHouseDefaults.USER.getEffectiveDefaultValue(), + (String) ClickHouseDefaults.PASSWORD.getEffectiveDefaultValue())) + .build()); + + info = ClickHouseJdbcUrlParser.parse("jdbc:clickhouse://localhost:4321/ndb", null); + Assert.assertEquals(info.getUri(), new URI("jdbc:clickhouse:http://localhost:4321/ndb")); + Assert.assertEquals(info.getServer(), + ClickHouseNode.builder().host("localhost").port(ClickHouseProtocol.HTTP, 4321).database("ndb") + .credentials(ClickHouseCredentials.fromUserAndPassword( + (String) ClickHouseDefaults.USER.getEffectiveDefaultValue(), + (String) ClickHouseDefaults.PASSWORD.getEffectiveDefaultValue())) + .build()); + + Properties props = new Properties(); + props.setProperty("database", "db1"); + info = ClickHouseJdbcUrlParser.parse("jdbc:clickhouse://me@localhost:1234/mydb?password=123", props); + Assert.assertEquals(info.getUri(), new URI("jdbc:clickhouse:http://localhost:1234/db1")); + Assert.assertEquals(info.getServer(), + ClickHouseNode.builder().host("localhost").port(ClickHouseProtocol.HTTP, 1234).database("db1") + .credentials(ClickHouseCredentials.fromUserAndPassword("me", "123")).build()); + Assert.assertEquals(info.getProperties().getProperty("database"), "db1"); + } + + @Test(groups = "unit") + public void testParseCredentials() throws Exception { + Properties props = new Properties(); + props.setProperty("user", "default1"); + props.setProperty("password", "password1"); + ClickHouseNode server = ClickHouseJdbcUrlParser.parse("jdbc:clickhouse://user:a:passwd@foo.ch/test", props) + .getServer(); + Assert.assertEquals(server.getCredentials().get().getUserName(), "default1"); + Assert.assertEquals(server.getCredentials().get().getPassword(), "password1"); + + server = ClickHouseJdbcUrlParser.parse("jdbc:clickhouse://let%40me%3Ain:let%40me%3Ain@foo.ch", null) + .getServer(); + Assert.assertEquals(server.getCredentials().get().getUserName(), "let@me:in"); + Assert.assertEquals(server.getCredentials().get().getPassword(), "let@me:in"); + } +} diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/FakeTransactionTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/FakeTransactionTest.java new file mode 100644 index 000000000..27fc6c13d --- /dev/null +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/FakeTransactionTest.java @@ -0,0 +1,100 @@ +package com.clickhouse.jdbc.internal; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collections; + +import com.clickhouse.jdbc.internal.FakeTransaction.FakeSavepoint; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class FakeTransactionTest { + @Test(groups = "unit") + public void testQuery() { + FakeTransaction tx = new FakeTransaction(); + Assert.assertNotNull(tx.id); + Assert.assertEquals(tx.getQueries(), Collections.emptyList()); + Assert.assertEquals(tx.getSavepoints(), Collections.emptyList()); + + String queryId = tx.newQuery(null); + Assert.assertNotNull(queryId); + Assert.assertEquals(tx.getQueries(), Collections.singleton(queryId)); + Assert.assertEquals(tx.getSavepoints(), Collections.emptyList()); + + String newQueryId = tx.newQuery(queryId); + Assert.assertNotNull(newQueryId); + Assert.assertNotEquals(newQueryId, queryId); + Assert.assertEquals(tx.getQueries(), Arrays.asList(queryId, newQueryId)); + Assert.assertEquals(tx.getSavepoints(), Collections.emptyList()); + + tx.clear(); + Assert.assertEquals(tx.getQueries(), Collections.emptyList()); + Assert.assertEquals(tx.getSavepoints(), Collections.emptyList()); + + queryId = tx.newQuery(""); + Assert.assertNotNull(queryId); + Assert.assertEquals(tx.getQueries(), Collections.singleton(queryId)); + Assert.assertEquals(tx.getSavepoints(), Collections.emptyList()); + } + + @Test(groups = "unit") + public void testSavepoint() throws SQLException { + FakeTransaction tx = new FakeTransaction(); + Assert.assertNotNull(tx.id); + Assert.assertEquals(tx.getQueries(), Collections.emptyList()); + Assert.assertEquals(tx.getSavepoints(), Collections.emptyList()); + + FakeSavepoint unnamedSavepoint = tx.newSavepoint(null); + FakeSavepoint s1 = unnamedSavepoint; + Assert.assertEquals(unnamedSavepoint.id, 0); + Assert.assertEquals(unnamedSavepoint.getSavepointId(), 0); + Assert.assertNull(unnamedSavepoint.name, "Un-named savepoint should not have name"); + Assert.assertThrows(SQLException.class, () -> s1.getSavepointName()); + Assert.assertEquals(tx.getQueries(), Collections.emptyList()); + Assert.assertEquals(tx.getSavepoints(), Collections.singleton(unnamedSavepoint)); + + FakeSavepoint namedSavepoint = tx.newSavepoint("tmp"); + FakeSavepoint s2 = namedSavepoint; + Assert.assertEquals(namedSavepoint.id, 0); + Assert.assertThrows(SQLException.class, () -> s2.getSavepointId()); + Assert.assertEquals(namedSavepoint.name, "tmp"); + Assert.assertEquals(namedSavepoint.getSavepointName(), "tmp"); + Assert.assertEquals(tx.getQueries(), Collections.emptyList()); + Assert.assertEquals(tx.getSavepoints(), Arrays.asList(unnamedSavepoint, namedSavepoint)); + + tx.toSavepoint(namedSavepoint); + Assert.assertEquals(tx.getQueries(), Collections.emptyList()); + Assert.assertEquals(tx.getSavepoints(), Collections.singleton(unnamedSavepoint)); + + tx.toSavepoint(unnamedSavepoint); + Assert.assertEquals(tx.getQueries(), Collections.emptyList()); + Assert.assertEquals(tx.getSavepoints(), Collections.emptyList()); + + tx.clear(); + Assert.assertEquals(tx.getQueries(), Collections.emptyList()); + Assert.assertEquals(tx.getSavepoints(), Collections.emptyList()); + + String queryId = tx.newQuery(null); + FakeSavepoint s3 = unnamedSavepoint = tx.newSavepoint(null); + Assert.assertEquals(unnamedSavepoint.id, 1); + Assert.assertEquals(unnamedSavepoint.getSavepointId(), 1); + Assert.assertNull(unnamedSavepoint.name, "Un-named savepoint should not have name"); + Assert.assertThrows(SQLException.class, () -> s3.getSavepointName()); + Assert.assertEquals(tx.getQueries().size(), 1); + Assert.assertEquals(tx.getSavepoints(), Collections.singleton(unnamedSavepoint)); + + tx.newQuery(null); + FakeSavepoint s4 = namedSavepoint = tx.newSavepoint("tmp"); + Assert.assertEquals(namedSavepoint.id, 2); + Assert.assertThrows(SQLException.class, () -> s4.getSavepointId()); + Assert.assertEquals(namedSavepoint.name, "tmp"); + Assert.assertEquals(namedSavepoint.getSavepointName(), "tmp"); + Assert.assertEquals(tx.getQueries().size(), 2); + Assert.assertEquals(tx.getSavepoints(), Arrays.asList(unnamedSavepoint, namedSavepoint)); + + tx.toSavepoint(unnamedSavepoint); + Assert.assertEquals(tx.getQueries(), Collections.singleton(queryId)); + Assert.assertEquals(tx.getSavepoints(), Collections.emptyList()); + } +} diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlParserTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/parser/ClickHouseSqlParserTest.java similarity index 94% rename from clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlParserTest.java rename to clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/parser/ClickHouseSqlParserTest.java index 77abc0503..9dcbb3d72 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlParserTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/parser/ClickHouseSqlParserTest.java @@ -1,9 +1,7 @@ -package ru.yandex.clickhouse.jdbc.parser; +package com.clickhouse.jdbc.parser; import org.testng.annotations.Test; -import ru.yandex.clickhouse.settings.ClickHouseProperties; - import static org.testng.Assert.assertEquals; import java.io.BufferedReader; @@ -18,9 +16,11 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.clickhouse.client.ClickHouseConfig; + public class ClickHouseSqlParserTest { private ClickHouseSqlStatement[] parse(String sql) { - return ClickHouseSqlParser.parse(sql, new ClickHouseProperties()); + return ClickHouseSqlParser.parse(sql, new ClickHouseConfig()); } private String loadSql(String file) { @@ -264,14 +264,14 @@ public void testRevokeStatement() { public void testSelectStatement() { String sql; - assertEquals(parse(sql = "select\n1"), new ClickHouseSqlStatement[] { - new ClickHouseSqlStatement(sql, StatementType.SELECT, null, null, "unknown", null, null, null, null) }); - assertEquals(parse(sql = "select\r\n1"), new ClickHouseSqlStatement[] { - new ClickHouseSqlStatement(sql, StatementType.SELECT, null, null, "unknown", null, null, null, null) }); + assertEquals(parse(sql = "select\n1"), new ClickHouseSqlStatement[] { new ClickHouseSqlStatement(sql, + StatementType.SELECT, null, null, "unknown", null, null, null, null, null, null) }); + assertEquals(parse(sql = "select\r\n1"), new ClickHouseSqlStatement[] { new ClickHouseSqlStatement(sql, + StatementType.SELECT, null, null, "unknown", null, null, null, null, null, null) }); assertEquals(parse(sql = "select 314 limit 5\nFORMAT JSONCompact;"), new ClickHouseSqlStatement[] { new ClickHouseSqlStatement("select 314 limit 5\nFORMAT JSONCompact", - StatementType.SELECT, null, null, "unknown", "JSONCompact", null, null, null) }); + StatementType.SELECT, null, null, "unknown", null, "JSONCompact", null, null, null, null) }); checkSingleStatement(parse(sql = "select (())"), sql, StatementType.SELECT); checkSingleStatement(parse(sql = "select []"), sql, StatementType.SELECT); @@ -307,16 +307,17 @@ public void testSelectStatement() { assertEquals(parse(sql = loadSql("issue-441_with-totals.sql")), new ClickHouseSqlStatement[] { new ClickHouseSqlStatement(sql, StatementType.SELECT, null, null, - "unknown", null, null, null, new HashMap() { + "unknown", null, null, null, null, new HashMap() { { put("TOTALS", 208); } - }) }); + }, null) }); assertEquals(parse(sql = loadSql("issue-555_custom-format.sql")), new ClickHouseSqlStatement[] { new ClickHouseSqlStatement(sql, StatementType.SELECT, null, null, "wrd", - "CSVWithNames", null, null, null) }); - assertEquals(parse(sql = loadSql("with-clause.sql")), new ClickHouseSqlStatement[] { - new ClickHouseSqlStatement(sql, StatementType.SELECT, null, null, "unknown", null, null, null, null) }); + null, "CSVWithNames", null, null, null, null) }); + assertEquals(parse(sql = loadSql("with-clause.sql")), + new ClickHouseSqlStatement[] { new ClickHouseSqlStatement(sql, StatementType.SELECT, null, null, + "unknown", null, null, null, null, null, null) }); } @Test(groups = "unit") @@ -358,9 +359,10 @@ public void testTruncateStatement() { public void testUpdateStatement() { String sql; - checkSingleStatement(parse(sql = "update a set a='1'"), sql, StatementType.UPDATE); + checkSingleStatement(parse(sql = "update a set a='1'"), sql, StatementType.UPDATE, + ClickHouseSqlStatement.DEFAULT_DATABASE, "a"); checkSingleStatement(parse(sql = "update a.a set `a`=2 where upper(a)=upper(lower(b))"), sql, - StatementType.UPDATE); + StatementType.UPDATE, "a", "a"); } @Test(groups = "unit") @@ -390,13 +392,14 @@ public void testComments() throws ParseException { @Test(groups = "unit") public void testMultipleStatements() throws ParseException { - assertEquals(parse("use ab;;;select 1; ;\t;\r;\n"), new ClickHouseSqlStatement[] { - new ClickHouseSqlStatement("use ab", StatementType.USE, null, "ab", null, null, null, null, null), - new ClickHouseSqlStatement("select 1", StatementType.SELECT) }); + assertEquals(parse("use ab;;;select 1; ;\t;\r;\n"), + new ClickHouseSqlStatement[] { new ClickHouseSqlStatement("use ab", StatementType.USE, null, "ab", null, + null, null, null, null, null, null), + new ClickHouseSqlStatement("select 1", StatementType.SELECT) }); assertEquals(parse("select * from \"a;1\".`b;c`;;;select 1 as `a ; a`; ;\t;\r;\n"), new ClickHouseSqlStatement[] { new ClickHouseSqlStatement("select * from \"a;1\".`b;c`", StatementType.SELECT, null, "a;1", - "b;c", null, null, null, null), + "b;c", null, null, null, null, null, null), new ClickHouseSqlStatement("select 1 as `a ; a`", StatementType.SELECT) }); } @@ -513,7 +516,7 @@ public void testParameterHandling() throws ParseException { assertEquals(stmts.length, 1); assertEquals(stmts[0].getSQL(), sql); - stmts = ClickHouseSqlParser.parse(sql, new ClickHouseProperties(), new ParseHandler() { + stmts = ClickHouseSqlParser.parse(sql, new ClickHouseConfig(), new ParseHandler() { @Override public String handleParameter(String cluster, String database, String table, int columnIndex) { return String.valueOf(columnIndex); @@ -530,7 +533,7 @@ public void testMacroHandling() throws ParseException { assertEquals(stmts.length, 1); assertEquals(stmts[0].getSQL(), "select from ()"); - stmts = ClickHouseSqlParser.parse(sql, new ClickHouseProperties(), new ParseHandler() { + stmts = ClickHouseSqlParser.parse(sql, new ClickHouseConfig(), new ParseHandler() { @Override public String handleMacro(String name, List parameters) { if ("listOfColumns".equals(name)) { diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlUtilsTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/parser/ClickHouseSqlUtilsTest.java similarity index 98% rename from clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlUtilsTest.java rename to clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/parser/ClickHouseSqlUtilsTest.java index f6347b685..2a2b804d2 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlUtilsTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/parser/ClickHouseSqlUtilsTest.java @@ -1,4 +1,4 @@ -package ru.yandex.clickhouse.jdbc.parser; +package com.clickhouse.jdbc.parser; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/ClickhouseJdbcUrlParserTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/ClickhouseJdbcUrlParserTest.java index e4df50e74..d650c38a4 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/ClickhouseJdbcUrlParserTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/ClickhouseJdbcUrlParserTest.java @@ -4,7 +4,6 @@ import com.clickhouse.client.ClickHouseProtocol; -import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V2.Tag.Protocol; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/JdbcIntegrationTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/JdbcIntegrationTest.java index b5511fdc0..9a3408836 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/JdbcIntegrationTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/JdbcIntegrationTest.java @@ -12,11 +12,21 @@ import com.clickhouse.client.ClickHouseProtocol; public abstract class JdbcIntegrationTest extends BaseIntegrationTest { + private static final String CLASS_PREFIX = "ClickHouse"; + private static final String CLASS_SUFFIX = "Test"; + protected final String dbName; public JdbcIntegrationTest() { String className = getClass().getSimpleName(); - this.dbName = className.endsWith("Test") ? className.substring(0, className.length() - 5) : className; + if (className.startsWith(CLASS_PREFIX)) { + className = className.substring(CLASS_PREFIX.length()); + } + if (className.endsWith(CLASS_SUFFIX)) { + className = className.substring(0, className.length() - CLASS_SUFFIX.length()); + } + + this.dbName = "test_" + className.toLowerCase(); } public String getClickHouseHttpAddress() { diff --git a/clickhouse-jdbc/src/test/resources/log4j.properties b/clickhouse-jdbc/src/test/resources/log4j.properties index 04d4e6be0..c9e52c9ee 100644 --- a/clickhouse-jdbc/src/test/resources/log4j.properties +++ b/clickhouse-jdbc/src/test/resources/log4j.properties @@ -1,5 +1,5 @@ log4j.rootLogger=WARN, STDOUT -log4j.category.ru.yandex.clickhouse=DEBUG +log4j.category.com.clickhouse=DEBUG #log4j.logger.org.apache.http=DEBUG log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout diff --git a/pom.xml b/pom.xml index 80c8208a3..d60039fe9 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,11 @@ 1.16.0 7.4.0 - 1.0.0 + 2.7.4 + 8.0.27 + 42.3.1 + + 1.1.0 3.3.0 3.8.1 @@ -103,6 +107,7 @@ 3.0.0-M3 3.0.0-M5 1.2.7 + 4.9.9 1.6 3.2.0 0.8.6 @@ -254,6 +259,40 @@ testng ${testng.version} + + + org.mariadb.jdbc + mariadb-java-client + ${mariadb-driver.version} + + + * + * + + + + + mysql + mysql-connector-java + ${mysql-driver.version} + + + * + * + + + + + org.postgresql + postgresql + ${postgresql-driver.version} + + + * + * + + + @@ -407,6 +446,29 @@ + + io.github.git-commit-id + git-commit-id-maven-plugin + ${git-plugin.version} + + + get-the-git-infos + + revision + + initialize + + + + full + false + + ^git.build.(time|version)$ + ^git.commit.id.(abbrev|full)$ + + true + + @@ -626,6 +688,10 @@ + + io.github.git-commit-id + git-commit-id-maven-plugin + org.apache.maven.plugins maven-jar-plugin @@ -636,6 +702,7 @@ true + ${git.commit.id.full}