diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 669052292..49700cd17 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -26,7 +26,7 @@ on: driver: description: "Driver version" required: true - default: "0.3.0-SNAPSHOT" + default: "0.3.1-SNAPSHOT" options: description: "Benchmark options" required: true @@ -62,7 +62,7 @@ jobs: mvn --batch-mode --update-snapshots -q -DskipTests install cd clickhouse-benchmark mvn --batch-mode --update-snapshots install - java -jar target/benchmarks.jar -rf text -p client=clickhouse-jdbc Basic + java -DclickhouseVersion="21.3" -jar target/benchmarks.jar -rf text -p client=clickhouse-jdbc Basic echo "BENCHMARK_REPORT<> $GITHUB_ENV cat jmh-result.text >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV @@ -110,7 +110,7 @@ jobs: run: | mvn --batch-mode --update-snapshots -DskipTests -pl clickhouse-benchmark -am package cd clickhouse-benchmark - java -jar target/benchmarks.jar -rf json ${{ github.event.inputs.options }} > output.txt + java -DclickhouseVersion="21.3" -jar target/benchmarks.jar -rf json ${{ github.event.inputs.options }} > output.txt echo "BENCHMARK_REPORT<> $GITHUB_ENV tail -n +$(grep -n '^REMEMBER:' output.txt | tail -1 | awk -F: '{print $1+6}') output.txt | head -n -2 | grep -v ':ยท' >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e8b53f233..f003027ef 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,7 @@ on: version: description: "Release version" required: true - default: "0.3.0-SNAPSHOT" + default: "0.3.1-SNAPSHOT" jobs: release: diff --git a/.github/workflows/timezone.yml b/.github/workflows/timezone.yml index 78fd5f1f5..6134c5b15 100644 --- a/.github/workflows/timezone.yml +++ b/.github/workflows/timezone.yml @@ -59,4 +59,5 @@ jobs: find . -type f -name "pom.xml" -exec sed -i -e 's|.*argLine.*timezone=.*||g' '{}' \; mvn --batch-mode --update-snapshots \ -DclickhouseTimezone=${{ matrix.serverTz }} \ + -DclickhouseVersion=21.3 \ -Duser.timezone=${{ matrix.clientTz }} verify diff --git a/.gitignore b/.gitignore index 507b39f58..8e4ab4f9f 100644 --- a/.gitignore +++ b/.gitignore @@ -29,10 +29,14 @@ log/ target/ # Generated files +.flattened-pom.xml **/parser/*CharStream.java -**/parser/ClickHouseSqlParser*.java +**/parser/ClickHouseSqlParser.java +**/parser/ClickHouseSqlParserConstants.java +**/parser/ClickHouseSqlParserTokenManager.java **/parser/Token*.java **/parser/ParseException.java +jmh-result.* # Shell scripts *.sh diff --git a/CHANGELOG b/CHANGELOG index 8c4ff5840..e9c0034ed 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,15 @@ +0.3.1 + * BREAKING CHANGE - move query from url to request body + * BREAKING CHANGE - always parse SQL(use extended API to skip that) + * BREAKING CHANGE - remove keepAliveTimeout and useNewParser from ClickHouseProperties + * BREAKING CHANGE - exclude RoaringBitmap from shaded jar + * add new connection setting useSharedCookieStore for load balancing + * add new query parameters: allow_experimental_bigint_types, allow_experimental_map_type, and join_algorithm + * add new format: CustomSeparated and RowBinaryWithNamesAndTypes + * fix 400 bad request error when dealing with large query + * fix parser issue when DESC statement contains alias + * support batch processing with arbitrary query - update and delete are not recommended so there'll be warnings + * support multi-statement sql - session will be used automatically and only the last result will be returned 0.3.0 * BREAKING CHANGE - dropped JDK 7 support * BREAKING CHANGE - removed Guava dependency(and so is UnsignedLong) diff --git a/README.md b/README.md index 7bc8c2536..b615e9ed8 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ It has support of a minimal subset of features to be usable. ru.yandex.clickhouse clickhouse-jdbc - 0.3.0 + 0.3.1 ``` @@ -20,8 +20,31 @@ URL syntax: JDBC Driver Class: `ru.yandex.clickhouse.ClickHouseDriver` -additionally, if you have a few instances, you can use `BalancedClickhouseDataSource`. +For example: +```java +String url = "jdbc:clickhouse://localhost:8123/test"; +ClickHouseProperties properties = new ClickHouseProperties(); +// set connection options - see more defined in ClickHouseConnectionSettings +properties.setClientName("Agent #1"); +... +// set default request options - more in ClickHouseQueryParam +properties.setSessionId("default-session-id"); +... + +ClickHouseDataSource dataSource = new ClickHouseDataSource(url, properties) +String sql = "select * from mytable"; +Map additionalDBParams = new HashMap<>(); +// set request options, which will override the default ones in ClickHouseProperties +additionalDBParams.put(ClickHouseQueryParam.SESSION_ID, "new-session-id"); +... +try (ClickHouseConnection conn = dataSource.getConnection(); + ClickHouseStatement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(sql, additionalDBParams)) { + ... +} +``` +Additionally, if you have a few instances, you can use `BalancedClickhouseDataSource`. ### Extended API In order to provide non-JDBC complaint data manipulation functionality, proprietary API exists. diff --git a/clickhouse-benchmark/pom.xml b/clickhouse-benchmark/pom.xml index c10861b59..fc02afda9 100644 --- a/clickhouse-benchmark/pom.xml +++ b/clickhouse-benchmark/pom.xml @@ -8,13 +8,13 @@ ${revision} - ${parent.groupId} clickhouse-benchmark ${revision} jar - clickhouse-benchmark + ${project.artifactId} Benchmarks for ClickHouse clients + https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-benchmark 1.4.4 diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java index 62d44bf6b..13d787529 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java @@ -33,8 +33,7 @@ public void doSetup(ServerState serverState) throws Exception { serverState.getUser(), serverState.getPassword()), new Properties()); try (Statement s = conn.createStatement()) { - s.execute( - "create table if not exists test_insert(i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory"); + s.execute("create table if not exists system.test_insert(i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory"); } } catch (SQLException e) { e.printStackTrace(); @@ -43,9 +42,9 @@ public void doSetup(ServerState serverState) throws Exception { } @TearDown(Level.Trial) - public void doTearDown() throws SQLException { + public void doTearDown(ServerState serverState) throws SQLException { try (Statement s = conn.createStatement()) { - s.execute("drop table if exists test_insert"); + s.execute("drop table if exists system.test_insert"); } conn.close(); } diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java index 44eea7efe..b6b2cfe17 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java @@ -19,7 +19,7 @@ public int insert10kUInt64Rows(ClientState state) throws Throwable { final int rows = 10000; final int num = new Random().nextInt(rows); - return executeInsert(state, "insert into test_insert(i) values(?)", new Enumeration() { + return executeInsert(state, "insert into system.test_insert(i) values(?)", new Enumeration() { int counter = 0; @Override @@ -39,7 +39,7 @@ public int insert10kStringRows(ClientState state) throws Throwable { final int rows = 10000; final int num = new Random().nextInt(rows); - return executeInsert(state, "insert into test_insert(s) values(?)", new Enumeration() { + return executeInsert(state, "insert into system.test_insert(s) values(?)", new Enumeration() { int counter = 0; @Override @@ -59,7 +59,7 @@ public int insert10kTimestampRows(ClientState state) throws Throwable { final int rows = 10000; final int num = new Random().nextInt(rows); - return executeInsert(state, "insert into test_insert(t) values(?)", new Enumeration() { + return executeInsert(state, "insert into system.test_insert(t) values(?)", new Enumeration() { int counter = 0; @Override diff --git a/clickhouse-client/pom.xml b/clickhouse-client/pom.xml index 32b882d42..6045af1f1 100644 --- a/clickhouse-client/pom.xml +++ b/clickhouse-client/pom.xml @@ -8,20 +8,94 @@ ${revision} - ${parent.groupId} clickhouse-client ${revision} jar + ${project.artifactId} + Unified Java client for ClickHouse + https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-client + + + 4.1.4 + + + + com.github.ben-manes.caffeine + caffeine + provided + + + com.lmax + disruptor + provided + + + dnsjava + dnsjava + provided + + + org.roaringbitmap + RoaringBitmap + provided + org.slf4j slf4j-api + + + org.slf4j + slf4j-log4j12 + test + + + org.mockito + mockito-all + test + + + com.github.tomakehurst + wiremock-jre8 + test + + + org.testcontainers + testcontainers + test + + + org.testng + testng + test + + + com.helger.maven + ph-javacc-maven-plugin + ${javacc-plugin.version} + + + jjc + generate-sources + + javacc + + + ${jdk.version} + true + tech.clickhouse.client.parser + src/main/javacc + src/main/java + + + + org.apache.maven.plugins maven-compiler-plugin @@ -31,10 +105,32 @@ true -Xlint:all - -Werror + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + env_str + 416 + false + + + diff --git a/clickhouse-grpc-client/pom.xml b/clickhouse-grpc-client/pom.xml index 41cb9243f..796d2ec0f 100644 --- a/clickhouse-grpc-client/pom.xml +++ b/clickhouse-grpc-client/pom.xml @@ -8,14 +8,17 @@ ${revision} - ${parent.groupId} clickhouse-grpc-client ${revision} jar + ${project.artifactId} + gRPC client for ClickHouse + https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-grpc-client + - ${parent.groupId} + ${project.parent.groupId} clickhouse-client ${revision} diff --git a/clickhouse-http-client/pom.xml b/clickhouse-http-client/pom.xml index a89830997..c8f3289c0 100644 --- a/clickhouse-http-client/pom.xml +++ b/clickhouse-http-client/pom.xml @@ -8,16 +8,56 @@ ${revision} - ${parent.groupId} clickhouse-http-client ${revision} jar + ${project.artifactId} + HTTP client for ClickHouse + https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-http-client + + + 4.5.13 + + - ${parent.groupId} + ${project.parent.groupId} + clickhouse-client + ${revision} + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + org.apache.httpcomponents + httpmime + ${httpclient.version} + + + + ${project.parent.groupId} clickhouse-client ${revision} + test-jar + test + + + org.slf4j + slf4j-log4j12 + test + + + org.testcontainers + testcontainers + test + + + org.testng + testng + test @@ -32,10 +72,18 @@ true -Xlint:all - -Werror + + + org.apache.maven.plugins + maven-failsafe-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + diff --git a/clickhouse-jdbc/pom.xml b/clickhouse-jdbc/pom.xml index b57b275b0..bc752ac94 100644 --- a/clickhouse-jdbc/pom.xml +++ b/clickhouse-jdbc/pom.xml @@ -13,6 +13,10 @@ ${revision} jar + ${project.artifactId} + JDBC driver for ClickHouse + https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-jdbc + serebrserg @@ -50,6 +54,13 @@ + org.apache.httpcomponents httpclient @@ -73,8 +84,9 @@ jackson-databind - com.github.RoaringBitmap + org.roaringbitmap RoaringBitmap + provided org.slf4j @@ -202,10 +214,6 @@ net.jpountz ${shade.base}.jpountz - - org.roaringbitmap - ${shade.base}.bitmap - org.slf4j ${shade.base}.slf4j diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnection.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnection.java index 7454d1a09..78a9cb12f 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnection.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnection.java @@ -6,10 +6,6 @@ public interface ClickHouseConnection extends Connection { - - @Deprecated - ClickHouseStatement createClickHouseStatement() throws SQLException; - TimeZone getServerTimeZone(); TimeZone getTimeZone(); diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java index 4c557511a..6ec960ca4 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java @@ -122,12 +122,6 @@ public ClickHouseStatement createStatement(int resultSetType) throws SQLExceptio resultSetType)); } - @Deprecated - @Override - public ClickHouseStatement createClickHouseStatement() throws SQLException { - return createStatement(); - } - @Override public TimeZone getTimeZone() { return timezone; diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java index 27d4e37ec..4abb48f95 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java @@ -19,7 +19,6 @@ import java.sql.RowId; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; -import java.sql.SQLSyntaxErrorException; import java.sql.SQLXML; import java.sql.Time; import java.sql.Timestamp; @@ -31,15 +30,14 @@ import java.util.List; import java.util.Map; import java.util.TimeZone; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.http.entity.AbstractHttpEntity; import org.apache.http.impl.client.CloseableHttpClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import ru.yandex.clickhouse.jdbc.parser.ClickHouseSqlStatement; import ru.yandex.clickhouse.jdbc.parser.StatementType; - import ru.yandex.clickhouse.response.ClickHouseResponse; import ru.yandex.clickhouse.settings.ClickHouseProperties; import ru.yandex.clickhouse.settings.ClickHouseQueryParam; @@ -47,15 +45,14 @@ import ru.yandex.clickhouse.util.ClickHouseValueFormatter; public class ClickHousePreparedStatementImpl extends ClickHouseStatementImpl implements ClickHousePreparedStatement { + private static final Logger log = LoggerFactory.getLogger(ClickHousePreparedStatementImpl.class); static final String PARAM_MARKER = "?"; static final String NULL_MARKER = "\\N"; - private static final Pattern VALUES = Pattern.compile("(?i)VALUES[\\s]*\\("); - private final TimeZone dateTimeZone; private final TimeZone dateTimeTimeZone; - private final String sql; + private final ClickHouseSqlStatement parsedStmt; private final List sqlParts; private final ClickHousePreparedStatementParameter[] binds; private final List> parameterList; @@ -67,9 +64,14 @@ public ClickHousePreparedStatementImpl(CloseableHttpClient client, TimeZone serverTimeZone, int resultSetType) throws SQLException { super(client, connection, properties, resultSetType); - parseSingleStatement(sql); + parseSqlStatements(sql); + + if (parsedStmts.length != 1) { + throw new IllegalArgumentException("Only single statement is supported"); + } + + parsedStmt = parsedStmts[0]; - this.sql = sql; PreparedStatementParser parser = PreparedStatementParser.parse(sql, parsedStmt.getEndPosition(ClickHouseSqlStatement.KEYWORD_VALUES)); this.parameterList = parser.getParameters(); @@ -92,18 +94,19 @@ public void clearParameters() { @Override public ClickHouseResponse executeQueryClickhouseResponse() throws SQLException { - return super.executeQueryClickhouseResponse(buildSql()); + return executeQueryClickhouseResponse(buildSql(), null, null); } @Override public ClickHouseResponse executeQueryClickhouseResponse(Map additionalDBParams) throws SQLException { - return super.executeQueryClickhouseResponse(buildSql(), additionalDBParams); + return executeQueryClickhouseResponse(buildSql(), additionalDBParams, null); } - private String buildSql() throws SQLException { + private ClickHouseSqlStatement buildSql() throws SQLException { if (sqlParts.size() == 1) { - return sqlParts.get(0); + return new ClickHouseSqlStatement(sqlParts.get(0), parsedStmt.getStatementType()); } + checkBinded(); StringBuilder sb = new StringBuilder(sqlParts.get(0)); for (int i = 1, p = 0; i < sqlParts.size(); i++) { @@ -117,7 +120,7 @@ private String buildSql() throws SQLException { } sb.append(sqlParts.get(i)); } - return sb.toString(); + return new ClickHouseSqlStatement(sb.toString(), parsedStmt.getStatementType()); } private void checkBinded() throws SQLException { @@ -132,32 +135,34 @@ private void checkBinded() throws SQLException { @Override public boolean execute() throws SQLException { - return super.execute(buildSql()); + return executeQueryStatement(buildSql(), null, null, null) != null; } @Override public ResultSet executeQuery() throws SQLException { - return super.executeQuery(buildSql()); + return executeQueryStatement(buildSql(), null, null, null); } @Override public void clearBatch() throws SQLException { - batchRows.clear(); + super.clearBatch(); + + batchRows = new ArrayList<>(); } @Override public ResultSet executeQuery(Map additionalDBParams) throws SQLException { - return super.executeQuery(buildSql(), additionalDBParams); + return executeQuery(additionalDBParams, null); } @Override public ResultSet executeQuery(Map additionalDBParams, List externalData) throws SQLException { - return super.executeQuery(buildSql(), additionalDBParams, externalData); + return executeQueryStatement(buildSql(), additionalDBParams, externalData, null); } @Override public int executeUpdate() throws SQLException { - return super.executeUpdate(buildSql()); + return executeStatement(buildSql(), null, null, null); } private void setBind(int parameterIndex, String bind, boolean quote) { @@ -315,9 +320,18 @@ public void setObject(int parameterIndex, Object x) throws SQLException { } } + @Override + public void addBatch(String sql) throws SQLException { + throw new SQLException("addBatch(String) cannot be called in PreparedStatement or CallableStatement!"); + } + @Override public void addBatch() throws SQLException { - batchRows.addAll(buildBatch()); + if (parsedStmt.getStatementType() == StatementType.INSERT) { + batchRows.addAll(buildBatch()); + } else { + batchStmts.add(buildSql()); + } } private List buildBatch() throws SQLException { @@ -353,27 +367,28 @@ public int[] executeBatch() throws SQLException { @Override public int[] executeBatch(Map additionalDBParams) throws SQLException { int valuePosition = -1; - if (parsedStmt.getStatementType() == StatementType.INSERT && parsedStmt.hasValues()) { + String sql = parsedStmt.getSQL(); + StatementType type = parsedStmt.getStatementType(); + if (type == StatementType.INSERT && parsedStmt.hasValues()) { valuePosition = parsedStmt.getStartPosition(ClickHouseSqlStatement.KEYWORD_VALUES); - } else { - Matcher matcher = VALUES.matcher(sql); - if (matcher.find()) { - valuePosition = matcher.start(); - } } - if (valuePosition < 0) { - throw new SQLSyntaxErrorException( - "Query must be like 'INSERT INTO [db.]table [(c1, c2, c3)] VALUES (?, ?, ?)'. " + - "Got: " + sql - ); - } - String insertSql = sql.substring(0, valuePosition); - BatchHttpEntity entity = new BatchHttpEntity(batchRows); - sendStream(entity, insertSql, additionalDBParams); int[] result = new int[batchRows.size()]; Arrays.fill(result, 1); - batchRows = new ArrayList<>(); + if (valuePosition > 0) { // insert + String insertSql = sql.substring(0, valuePosition); + BatchHttpEntity entity = new BatchHttpEntity(batchRows); + sendStream(entity, insertSql, additionalDBParams); + } else { // others + if (type == StatementType.ALTER_DELETE || type == StatementType.ALTER_UPDATE) { + log.warn("UPDATE and DELETE should be used with caution, as they are expensive operations and not supposed to be used frequently."); + } else { + log.warn("PreparedStatement will slow down non-INSERT queries, so please consider to use Statement instead."); + } + result = super.executeBatch(); + } + + clearBatch(); return result; } @@ -443,7 +458,7 @@ public ResultSetMetaData getMetaData() throws SQLException { return currentResult.getMetaData(); } - if (!parsedStmt.isQuery() || (!parsedStmt.isRecognized() && !isSelect(sql))) { + if (!parsedStmt.isQuery()) { return null; } ResultSet myRs = executeQuery(Collections.singletonMap( @@ -624,9 +639,9 @@ private String getParameter(int paramIndex) { @Override public String asSql() { try { - return buildSql(); + return buildSql().getSQL(); } catch (SQLException e) { - return sql; + return parsedStmt.getSQL(); } } } diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java index 80dc0971c..695d9378c 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java @@ -74,7 +74,7 @@ ResultSet executeQuery(String sql, /** * Returns extended write-API, which simplifies uploading larger files or - * data streams + * data streams. * * @return a new {@link Writer} builder object which can be used to * construct a request to the server diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java index 906e2f166..7699f5833 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java @@ -3,6 +3,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; @@ -14,18 +15,20 @@ import java.util.EnumMap; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.TimeZone; import java.util.UUID; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; +import org.apache.http.StatusLine; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.AbstractHttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.entity.mime.MultipartEntityBuilder; @@ -40,7 +43,6 @@ import ru.yandex.clickhouse.except.ClickHouseExceptionSpecifier; import ru.yandex.clickhouse.jdbc.parser.ClickHouseSqlParser; import ru.yandex.clickhouse.jdbc.parser.ClickHouseSqlStatement; -import ru.yandex.clickhouse.jdbc.parser.StatementType; import ru.yandex.clickhouse.response.ClickHouseLZ4Stream; import ru.yandex.clickhouse.response.ClickHouseResponse; import ru.yandex.clickhouse.response.ClickHouseResponseSummary; @@ -58,6 +60,50 @@ public class ClickHouseStatementImpl extends ConfigurableApi batchStmts; /** * Current database name may be changed by {@link java.sql.Connection#setCatalog(String)} @@ -95,45 +143,156 @@ public class ClickHouseStatementImpl extends ConfigurableApi 0) { + stmt = parsedStmts[parsedStmts.length - 1]; + } + + return Objects.requireNonNull(stmt); + } + + protected void setLastStatement(ClickHouseSqlStatement stmt) { + if (parsedStmts != null && parsedStmts.length > 0) { + parsedStmts[parsedStmts.length - 1] = Objects.requireNonNull(stmt); + } + } + + protected ClickHouseSqlStatement[] parseSqlStatements(String sql) throws SQLException { + parsedStmts = ClickHouseSqlParser.parse(sql, properties); - if (stmts.length == 1) { - this.parsedStmt = stmts[0]; - } else { - this.parsedStmt = new ClickHouseSqlStatement(sql, StatementType.UNKNOWN); - // throw new SQLException("Multiple statements are not supported."); + if (parsedStmts == null || parsedStmts.length == 0) { + // should never happen + throw new IllegalArgumentException("Failed to parse given SQL: " + sql); } - if (this.parsedStmt.isIdemponent()) { - httpContext.setAttribute("is_idempotent", Boolean.TRUE); - } else { - httpContext.removeAttribute("is_idempotent"); + return parsedStmts; + } + + protected ClickHouseSqlStatement parseSqlStatements( + String sql, ClickHouseFormat preferredFormat, Map additionalDBParams) + throws SQLException { + parseSqlStatements(sql); + + // enable session when we have more than one statement + if (additionalDBParams != null && parsedStmts.length > 1 && properties.getSessionId() == null) { + additionalDBParams.put(ClickHouseQueryParam.SESSION_ID, UUID.randomUUID().toString()); } + + ClickHouseSqlStatement lastStmt = getLastStatement(); + ClickHouseSqlStatement formattedStmt = applyFormat(lastStmt, preferredFormat); + if (formattedStmt != lastStmt) { + setLastStatement(lastStmt = formattedStmt); + } + + return lastStmt; } - @Deprecated - private void parseSingleStatement(String sql, ClickHouseFormat preferredFormat) throws SQLException { - parseSingleStatement(sql); + protected ClickHouseSqlStatement applyFormat(ClickHouseSqlStatement stmt, ClickHouseFormat preferredFormat) { + if (Objects.requireNonNull(stmt).isQuery() && !stmt.hasFormat()) { + String sql = stmt.getSQL(); + String format = Objects.requireNonNull(preferredFormat).name(); - if (parsedStmt.isQuery() && !parsedStmt.hasFormat()) { - String format = preferredFormat.name(); Map positions = new HashMap<>(); - positions.putAll(parsedStmt.getPositions()); + positions.putAll(stmt.getPositions()); positions.put(ClickHouseSqlStatement.KEYWORD_FORMAT, sql.length()); - - sql = new StringBuilder(parsedStmt.getSQL()).append("\nFORMAT ").append(format).append(';') - .toString(); - parsedStmt = new ClickHouseSqlStatement(sql, parsedStmt.getStatementType(), - parsedStmt.getCluster(), parsedStmt.getDatabase(), parsedStmt.getTable(), - format, parsedStmt.getOutfile(), parsedStmt.getParameters(), positions); + + sql = new StringBuilder(sql).append("\nFORMAT ").append(format).toString(); + stmt = new ClickHouseSqlStatement(sql, stmt.getStatementType(), + stmt.getCluster(), stmt.getDatabase(), stmt.getTable(), + format, stmt.getOutfile(), stmt.getParameters(), positions); + } + + return stmt; + } + + protected Map importAdditionalDBParameters(Map additionalDBParams) { + if (additionalDBParams == null || additionalDBParams.isEmpty()) { + additionalDBParams = new EnumMap<>(ClickHouseQueryParam.class); + } else { // in case the given additionalDBParams is immutable + additionalDBParams = new EnumMap<>(additionalDBParams); + } + + return additionalDBParams; + } + + protected ResultSet updateResult(ClickHouseSqlStatement stmt, InputStream is) throws IOException, ClickHouseException { + ResultSet rs = null; + if (stmt.isQuery()) { + currentUpdateCount = -1; + currentResult = createResultSet( + properties.isCompress() ? new ClickHouseLZ4Stream(is) : is, properties.getBufferSize(), + stmt.getDatabaseOrDefault(properties.getDatabase()), + stmt.getTable(), + stmt.hasWithTotals(), + this, + getConnection().getTimeZone(), + properties + ); + currentResult.setMaxRows(maxRows); + rs = currentResult; + } else { + currentUpdateCount = 0; + try { + is.close(); + } catch (IOException e) { + log.error("can not close stream: {}", e.getMessage()); + } + } + + return rs; + } + + protected int executeStatement( + ClickHouseSqlStatement stmt, + Map additionalDBParams, + List externalData, + Map additionalRequestParams) throws SQLException { + additionalDBParams = importAdditionalDBParameters(additionalDBParams); + stmt = applyFormat(stmt, ClickHouseFormat.TabSeparatedWithNamesAndTypes); + + try (InputStream is = getInputStream(stmt, additionalDBParams, externalData, additionalRequestParams)) { + //noinspection StatementWithEmptyBody + } catch (IOException e) { + log.error("can not close stream: {}", e.getMessage()); + } + + return currentSummary != null ? (int) currentSummary.getWrittenRows() : 1; + } + + protected ResultSet executeQueryStatement(ClickHouseSqlStatement stmt, + Map additionalDBParams, + List externalData, + Map additionalRequestParams) throws SQLException { + additionalDBParams = importAdditionalDBParameters(additionalDBParams); + stmt = applyFormat(stmt, ClickHouseFormat.TabSeparatedWithNamesAndTypes); + + InputStream is = getInputStream(stmt, additionalDBParams, externalData, additionalRequestParams); + try { + return updateResult(stmt, is); + } catch (Exception e) { + try { + is.close(); + } catch (IOException ioe) { + log.error("can not close stream: {}", ioe.getMessage()); + } + throw ClickHouseExceptionSpecifier.specify(e, properties.getHost(), properties.getPort()); + } + } + + protected ClickHouseResponse executeQueryClickhouseResponse( + ClickHouseSqlStatement stmt, + Map additionalDBParams, + Map additionalRequestParams) throws SQLException { + additionalDBParams = importAdditionalDBParameters(additionalDBParams); + stmt = applyFormat(stmt, ClickHouseFormat.JSONCompact); + + try (InputStream is = getInputStream(stmt, additionalDBParams, null, additionalRequestParams)) { + return Jackson.getObjectMapper().readValue( + properties.isCompress() ? new ClickHouseLZ4Stream(is) : is, ClickHouseResponse.class); + } catch (IOException e) { + throw new RuntimeException(e); } } @@ -146,6 +305,8 @@ public ClickHouseStatementImpl(CloseableHttpClient client, ClickHouseConnection this.properties = properties == null ? new ClickHouseProperties() : properties; this.initialDatabase = this.properties.getDatabase(); this.isResultSetScrollable = (resultSetType != ResultSet.TYPE_FORWARD_ONLY); + + this.batchStmts = new ArrayList<>(); } @Override @@ -170,51 +331,17 @@ public ResultSet executeQuery(String sql, Map additionalRequestParams) throws SQLException { // forcibly disable extremes for ResultSet queries - if (additionalDBParams == null || additionalDBParams.isEmpty()) { - additionalDBParams = new EnumMap<>(ClickHouseQueryParam.class); - } else { - additionalDBParams = new EnumMap<>(additionalDBParams); - } + additionalDBParams = importAdditionalDBParameters(additionalDBParams); + // FIXME respect the value set in additionalDBParams? additionalDBParams.put(ClickHouseQueryParam.EXTREMES, "0"); - parseSingleStatement(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes); - if (!parsedStmt.isRecognized() && isSelect(sql)) { - Map positions = new HashMap<>(); - String dbName = extractDBName(sql); - String tableName = extractTableName(sql); - if (extractWithTotals(sql)) { - positions.put(ClickHouseSqlStatement.KEYWORD_TOTALS, 1); - } - parsedStmt = new ClickHouseSqlStatement(sql, StatementType.SELECT, - null, dbName, tableName, null, null, null, positions); - // httpContext.setAttribute("is_idempotent", Boolean.TRUE); - } + parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, additionalDBParams); - InputStream is = getInputStream(sql, additionalDBParams, externalData, additionalRequestParams); + InputStream is = getLastInputStream(additionalDBParams, externalData, additionalRequestParams); + ClickHouseSqlStatement parsedStmt = getLastStatement(); try { - if (parsedStmt.isQuery()) { - currentUpdateCount = -1; - currentResult = createResultSet(properties.isCompress() - ? new ClickHouseLZ4Stream(is) : is, properties.getBufferSize(), - parsedStmt.getDatabaseOrDefault(properties.getDatabase()), - parsedStmt.getTable(), - parsedStmt.hasWithTotals(), - this, - getConnection().getTimeZone(), - properties - ); - currentResult.setMaxRows(maxRows); - return currentResult; - } else { - currentUpdateCount = 0; - try { - is.close(); - } catch (IOException e) { - log.error("can not close stream: {}", e.getMessage()); - } - return null; - } + return updateResult(parsedStmt, is); } catch (Exception e) { try { is.close(); @@ -239,17 +366,12 @@ public ClickHouseResponse executeQueryClickhouseResponse(String sql, Map additionalDBParams, Map additionalRequestParams) throws SQLException { - parseSingleStatement(sql, ClickHouseFormat.JSONCompact); - if (parsedStmt.isRecognized()) { - sql = parsedStmt.getSQL(); - } else { - sql = addFormatIfAbsent(sql, ClickHouseFormat.JSONCompact); - } - - try (InputStream is = properties.isCompress() - ? new ClickHouseLZ4Stream(getInputStream(sql, additionalDBParams, null, additionalRequestParams)) - : getInputStream(sql, additionalDBParams, null, additionalRequestParams)) { - return Jackson.getObjectMapper().readValue(is, ClickHouseResponse.class); + additionalDBParams = importAdditionalDBParameters(additionalDBParams); + parseSqlStatements(sql, ClickHouseFormat.JSONCompact, additionalDBParams); + + try (InputStream is = getLastInputStream(additionalDBParams, null, additionalRequestParams)) { + return Jackson.getObjectMapper().readValue( + properties.isCompress() ? new ClickHouseLZ4Stream(is) : is, ClickHouseResponse.class); } catch (IOException e) { throw new RuntimeException(e); } @@ -267,30 +389,22 @@ public ClickHouseRowBinaryInputStream executeQueryClickhouseRowBinaryStream(Stri @Override public ClickHouseRowBinaryInputStream executeQueryClickhouseRowBinaryStream(String sql, Map additionalDBParams, Map additionalRequestParams) throws SQLException { - parseSingleStatement(sql, ClickHouseFormat.RowBinary); - if (parsedStmt.isRecognized()) { - sql = parsedStmt.getSQL(); - } else { - sql = addFormatIfAbsent(sql, ClickHouseFormat.RowBinary); - if (isSelect(sql)) { - parsedStmt = new ClickHouseSqlStatement(sql, StatementType.SELECT); - // httpContext.setAttribute("is_idempotent", Boolean.TRUE); - } else { - parsedStmt = new ClickHouseSqlStatement(sql, StatementType.UNKNOWN); - } - } + additionalDBParams = importAdditionalDBParameters(additionalDBParams); + parseSqlStatements(sql, ClickHouseFormat.RowBinaryWithNamesAndTypes, additionalDBParams); - InputStream is = getInputStream( - sql, + InputStream is = getLastInputStream( additionalDBParams, null, additionalRequestParams ); + ClickHouseSqlStatement parsedStmt = getLastStatement(); + try { if (parsedStmt.isQuery()) { currentUpdateCount = -1; + // FIXME get server timezone? currentRowBinaryResult = new ClickHouseRowBinaryInputStream(properties.isCompress() - ? new ClickHouseLZ4Stream(is) : is, getConnection().getTimeZone(), properties); + ? new ClickHouseLZ4Stream(is) : is, getConnection().getTimeZone(), properties, true); return currentRowBinaryResult; } else { currentUpdateCount = 0; @@ -313,9 +427,10 @@ public ClickHouseRowBinaryInputStream executeQueryClickhouseRowBinaryStream(Stri @Override public int executeUpdate(String sql) throws SQLException { - parseSingleStatement(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes); + Map additionalDBParams = new EnumMap<>(ClickHouseQueryParam.class); + parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, additionalDBParams); - try (InputStream is = getInputStream(sql, null, null, null)) { + try (InputStream is = getLastInputStream(additionalDBParams, null, null)) { //noinspection StatementWithEmptyBody } catch (IOException e) { log.error("can not close stream: {}", e.getMessage()); @@ -460,17 +575,27 @@ public int getResultSetType() throws SQLException { @Override public void addBatch(String sql) throws SQLException { - + for (ClickHouseSqlStatement s : ClickHouseSqlParser.parse(sql, properties)) { + this.batchStmts.add(s); + } } @Override public void clearBatch() throws SQLException { - + this.batchStmts = new ArrayList<>(); } @Override public int[] executeBatch() throws SQLException { - return new int[0]; + int len = batchStmts.size(); + int[] results = new int[len]; + for (int i = 0; i < len; i++) { + results[i] = executeStatement(batchStmts.get(i), null, null, null); + } + + clearBatch(); + + return results; } @Override @@ -556,124 +681,36 @@ public ClickHouseResponseSummary getResponseSummary() { return currentSummary; } - @Deprecated - static String clickhousifySql(String sql) { - return addFormatIfAbsent(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes); - } - - /** - * Adding FORMAT TabSeparatedWithNamesAndTypes if not added - * adds format only to select queries - */ - @Deprecated - private static String addFormatIfAbsent(final String sql, ClickHouseFormat format) { - String cleanSQL = sql.trim(); - if (!isSelect(cleanSQL)) { - return cleanSQL; - } - if (ClickHouseFormat.containsFormat(cleanSQL)) { - return cleanSQL; - } - StringBuilder sb = new StringBuilder(); - int idx = cleanSQL.endsWith(";") - ? cleanSQL.length() - 1 - : cleanSQL.length(); - sb.append(cleanSQL, 0, idx) - .append("\nFORMAT ") - .append(format.name()) - .append(';'); - return sb.toString(); - } - - @Deprecated - static boolean isSelect(String sql) { - for (int i = 0; i < sql.length(); i++) { - String nextTwo = sql.substring(i, Math.min(i + 2, sql.length())); - if ("--".equals(nextTwo)) { - i = Math.max(i, sql.indexOf("\n", i)); - } else if ("/*".equals(nextTwo)) { - i = Math.max(i, sql.indexOf("*/", i)); - } else if (Character.isLetter(sql.charAt(i))) { - String trimmed = sql.substring(i, Math.min(sql.length(), Math.max(i, sql.indexOf(" ", i)))); - for (String keyword : selectKeywords){ - if (trimmed.regionMatches(true, 0, keyword, 0, keyword.length())) { - return true; - } + private InputStream getLastInputStream( + Map additionalDBParams, + List externalData, + Map additionalRequestParams) throws ClickHouseException { + InputStream is = null; + for (int i = 0, len = parsedStmts.length; i < len; i++) { + // TODO skip useless queries to reduce network calls and server load + is = getInputStream(parsedStmts[i], additionalDBParams, externalData, additionalRequestParams); + // TODO multi-resultset + if (i + 1 < len) { + try { + is.close(); + } catch (IOException ioe) { + log.warn("Failed to close stream: {}", ioe.getMessage()); } - return false; - } - } - return false; - } - - @Deprecated - private String extractTableName(String sql) { - String s = extractDBAndTableName(sql); - if (s.contains(".")) { - return s.substring(s.indexOf(".") + 1); - } else { - return s; - } - } - - @Deprecated - private String extractDBName(String sql) { - String s = extractDBAndTableName(sql); - if (s.contains(".")) { - return s.substring(0, s.indexOf(".")); - } else { - return properties.getDatabase(); - } - } - - @Deprecated - private String extractDBAndTableName(String sql) { - if (Utils.startsWithIgnoreCase(sql, "select")) { - String withoutStrings = Utils.retainUnquoted(sql, '\''); - int fromIndex = withoutStrings.indexOf("from"); - if (fromIndex == -1) { - fromIndex = withoutStrings.indexOf("FROM"); - } - if (fromIndex != -1) { - String fromFrom = withoutStrings.substring(fromIndex); - String fromTable = fromFrom.substring("from".length()).trim(); - return fromTable.split(" ")[0]; } } - if (Utils.startsWithIgnoreCase(sql, "desc")) { - return "system.columns"; - } - if (Utils.startsWithIgnoreCase(sql, "show")) { - return "system.tables"; - } - return "system.unknown"; - } - @Deprecated - private boolean extractWithTotals(String sql) { - if (Utils.startsWithIgnoreCase(sql, "select")) { - String withoutStrings = Utils.retainUnquoted(sql, '\''); - return withoutStrings.toLowerCase(Locale.ROOT).contains(" with totals"); - } - return false; + return is; } private InputStream getInputStream( - String sql, + ClickHouseSqlStatement parsedStmt, Map additionalClickHouseDBParams, List externalData, Map additionalRequestParams ) throws ClickHouseException { - boolean ignoreDatabase = false; - if (parsedStmt.isRecognized()) { - sql = parsedStmt.getSQL(); - // TODO consider more scenarios like drop, show etc. - ignoreDatabase = parsedStmt.getStatementType() == StatementType.CREATE - && parsedStmt.containsKeyword(ClickHouseSqlStatement.KEYWORD_DATABASE); - } else { - sql = clickhousifySql(sql); - ignoreDatabase = sql.trim().regionMatches(true, 0, databaseKeyword, 0, databaseKeyword.length()); - } + String sql = parsedStmt.getSQL(); + boolean ignoreDatabase = parsedStmt.isRecognized() && !parsedStmt.isDML(); + log.debug("Executing SQL: {}", sql); additionalClickHouseDBParams = addQueryIdTo( @@ -681,27 +718,13 @@ private InputStream getInputStream( ? new EnumMap(ClickHouseQueryParam.class) : additionalClickHouseDBParams); - URI uri; - if (externalData == null || externalData.isEmpty()) { - uri = buildRequestUri( - null, - null, - additionalClickHouseDBParams, - additionalRequestParams, - ignoreDatabase - ); - } else { - // write sql in query params when there is external data - // as it is impossible to pass both external data and sql in body - // TODO move sql to request body when it is supported in clickhouse - uri = buildRequestUri( - sql, - externalData, - additionalClickHouseDBParams, - additionalRequestParams, - ignoreDatabase - ); - } + URI uri = buildRequestUri( + null, + externalData, + additionalClickHouseDBParams, + additionalRequestParams, + ignoreDatabase + ); log.debug("Request url: {}", uri); @@ -710,6 +733,7 @@ private InputStream getInputStream( requestEntity = new StringEntity(sql, StandardCharsets.UTF_8); } else { MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); + entityBuilder.addTextBody("query", sql); try { for (ClickHouseExternalData externalDataItem : externalData) { @@ -738,6 +762,12 @@ private InputStream getInputStream( HttpPost post = new HttpPost(uri); post.setEntity(requestEntity); + if (parsedStmt.isIdemponent()) { + httpContext.setAttribute("is_idempotent", Boolean.TRUE); + } else { + httpContext.removeAttribute("is_idempotent"); + } + HttpResponse response = client.execute(post, httpContext); entity = response.getEntity(); checkForErrorAndThrow(entity, response); @@ -784,6 +814,11 @@ URI buildRequestUri( ignoreDatabase ); + // avoid to reuse query id + if (additionalClickHouseDBParams != null) { + additionalClickHouseDBParams.remove(ClickHouseQueryParam.QUERY_ID); + } + return new URIBuilder() .setScheme(properties.getSsl() ? "https" : "http") .setHost(properties.getHost()) @@ -806,7 +841,7 @@ private List getUrlQueryParams( ) { List result = new ArrayList<>(); - if (sql != null) { + if (sql != null && !sql.isEmpty()) { result.add(new BasicNameValuePair("query", sql)); } @@ -1004,12 +1039,16 @@ public void sendStreamSQL(InputStream content, String sql) throws SQLException { void sendStream(Writer writer, HttpEntity content) throws ClickHouseException { HttpEntity entity = null; + // TODO no parser involved so user can execute arbitray statement here try { - - URI uri = buildRequestUri(writer.getSql(), null, writer.getAdditionalDBParams(), writer.getRequestParams(), false); + String sql = writer.getSql(); + boolean isContentCompressed = writer.getCompression() != ClickHouseCompression.none; + URI uri = buildRequestUri( + isContentCompressed ? sql : null, null, writer.getAdditionalDBParams(), writer.getRequestParams(), false); uri = followRedirects(uri); - content = applyRequestBodyCompression(content); + content = applyRequestBodyCompression( + new WrappedHttpEntity(isContentCompressed ? null : sql, content)); HttpPost httpPost = new HttpPost(uri); @@ -1036,7 +1075,8 @@ void sendStream(Writer writer, HttpEntity content) throws ClickHouseException { } private void checkForErrorAndThrow(HttpEntity entity, HttpResponse response) throws IOException, ClickHouseException { - if (response.getStatusLine().getStatusCode() != HttpURLConnection.HTTP_OK) { + StatusLine line = response.getStatusLine(); + if (line.getStatusCode() != HttpURLConnection.HTTP_OK) { InputStream messageStream = entity.getContent(); byte[] bytes = Utils.toByteArray(messageStream); if (properties.isCompress()) { @@ -1048,8 +1088,11 @@ private void checkForErrorAndThrow(HttpEntity entity, HttpResponse response) thr } } EntityUtils.consumeQuietly(entity); - String chMessage = new String(bytes, StandardCharsets.UTF_8); - throw ClickHouseExceptionSpecifier.specify(chMessage, properties.getHost(), properties.getPort()); + if (bytes.length == 0) { + throw ClickHouseExceptionSpecifier.specify(new IllegalStateException(line.toString()), properties.getHost(), properties.getPort()); + } else { + throw ClickHouseExceptionSpecifier.specify(new String(bytes, StandardCharsets.UTF_8), properties.getHost(), properties.getPort()); + } } } @@ -1086,6 +1129,7 @@ private Map addQueryIdTo(Map - * Tries to extract query parameters in a way that is usable for (batched) + * Parser for JDBC SQL Strings. + * + *

Tries to extract query parameters in a way that is usable for (batched) * prepared statements. */ final class PreparedStatementParser { @@ -88,7 +88,7 @@ private void parseSQL(String sql, int valuesEndPosition) { int quotedStart = 0; int partStart = 0; int sqlLength = sql.length(); - for (int i = valuesMode ? endPosition : 0, idxStart = i, idxEnd = i ; i < sqlLength; i++) { + for (int i = valuesMode ? endPosition : 0, idxStart = i, idxEnd = i; i < sqlLength; i++) { char c = sql.charAt(i); if (inSingleLineComment) { if (c == '\n') { diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java index a230d1a89..773914c05 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java @@ -13,7 +13,7 @@ * @author Dmitry Andreev */ public enum ClickHouseFormat { - + CustomSeparated, TabSeparated, TabSeparatedRaw, TabSeparatedWithNames, @@ -38,6 +38,7 @@ public enum ClickHouseFormat { PrettySpace, Protobuf, RowBinary, + RowBinaryWithNamesAndTypes, Native, Null, XML, diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultSet.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultSet.java index 944d06be6..07cac08de 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultSet.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultSet.java @@ -137,22 +137,20 @@ private static String[] toStringArray(ByteFragment headerFragment) { } /** - * Check if there is another row + * Check if there is another row. * * @return {@code true} if this result set has another row after the current * cursor position, {@code false} else - * @throws SQLException - * if something goes wrong - * @deprecated prefer to use JDBC API methods, for example {@link #isLast()} - * or simply looping using {@code while (rs.next())} + * @throws SQLException if something goes wrong */ - @Deprecated - public boolean hasNext() throws SQLException { + protected boolean hasNext() throws SQLException { if (nextLine == null && !lastReached) { try { nextLine = bis.next(); - if (nextLine == null || (maxRows != 0 && rowNumber >= maxRows) || (usesWithTotals && nextLine.length() == 0)) { + if (nextLine == null + || (maxRows != 0 && rowNumber >= maxRows) + || (usesWithTotals && nextLine.length() == 0)) { if (usesWithTotals) { if (onTheSeparatorRow()) { totalLine = bis.next(); diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseConnectionSettings.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseConnectionSettings.java index ed9bedadb..2f9b53568 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseConnectionSettings.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseConnectionSettings.java @@ -27,9 +27,6 @@ public enum ClickHouseConnectionSettings implements DriverPropertyCreator { + " ClickHouse rejects request execution if its time exceeds max_execution_time"), - @Deprecated - KEEP_ALIVE_TIMEOUT("keepAliveTimeout", 30 * 1000, ""), - /** * for ConnectionManager */ @@ -42,6 +39,7 @@ public enum ClickHouseConnectionSettings implements DriverPropertyCreator { * additional */ USE_OBJECTS_IN_ARRAYS("use_objects_in_arrays", false, "Whether Object[] should be used instead primitive arrays."), + USE_SHARED_COOKIE_STORE("useSharedCookieStore", false, "Whether to use shared cookie to store among all http clients of db are not"), MAX_COMPRESS_BUFFER_SIZE("maxCompressBufferSize", 1024*1024, ""), USE_SERVER_TIME_ZONE("use_server_time_zone", true, "Whether to use timezone from server. On connection init select timezone() will be executed"), @@ -51,8 +49,6 @@ public enum ClickHouseConnectionSettings implements DriverPropertyCreator { "If false, Date returned is a wrapper of a timestamp at start of the day in client timezone. " + "If true - at start of the day in server or use_timezone timezone."), CLIENT_NAME("client_name", "", "client_name or http_user_agent show up in system.query_log table, depending on the protocol you're using."), - @Deprecated - USE_NEW_PARSER("use_new_parser", true, "Whether to use JavaCC based SQL parser or not.") ; private final String key; diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java index b99b476e1..10da23b81 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java @@ -17,8 +17,6 @@ public class ClickHouseProperties { private int socketTimeout; private int connectionTimeout; private int dataTransferTimeout; - @Deprecated - private int keepAliveTimeout; private int timeToLiveMillis; private int defaultMaxPerRoute; private int maxTotal; @@ -56,6 +54,8 @@ public class ClickHouseProperties { private String useTimeZone; private boolean useServerTimeZoneForDates; private boolean useObjectsInArrays; + // the shared cookie store is scoped to a database + private boolean useSharedCookieStore; // queries settings private Integer maxParallelReplicas; @@ -98,8 +98,6 @@ public class ClickHouseProperties { private Boolean sendProgressInHttpHeaders; private Boolean waitEndOfQuery; private String clientName; - @Deprecated - private boolean useNewParser; public ClickHouseProperties() { this(new Properties()); @@ -113,7 +111,6 @@ public ClickHouseProperties(Properties info) { this.socketTimeout = (Integer)getSetting(info, ClickHouseConnectionSettings.SOCKET_TIMEOUT); this.connectionTimeout = (Integer)getSetting(info, ClickHouseConnectionSettings.CONNECTION_TIMEOUT); this.dataTransferTimeout = (Integer)getSetting(info, ClickHouseConnectionSettings.DATA_TRANSFER_TIMEOUT); - this.keepAliveTimeout = (Integer)getSetting(info, ClickHouseConnectionSettings.KEEP_ALIVE_TIMEOUT); this.timeToLiveMillis = (Integer)getSetting(info, ClickHouseConnectionSettings.TIME_TO_LIVE_MILLIS); this.defaultMaxPerRoute = (Integer)getSetting(info, ClickHouseConnectionSettings.DEFAULT_MAX_PER_ROUTE); this.maxTotal = (Integer)getSetting(info, ClickHouseConnectionSettings.MAX_TOTAL); @@ -130,8 +127,8 @@ public ClickHouseProperties(Properties info) { this.useTimeZone = (String)getSetting(info, ClickHouseConnectionSettings.USE_TIME_ZONE); this.useServerTimeZoneForDates = (Boolean)getSetting(info, ClickHouseConnectionSettings.USE_SERVER_TIME_ZONE_FOR_DATES); this.useObjectsInArrays = (Boolean)getSetting(info, ClickHouseConnectionSettings.USE_OBJECTS_IN_ARRAYS); + this.useSharedCookieStore = (Boolean)getSetting(info, ClickHouseConnectionSettings.USE_SHARED_COOKIE_STORE); this.clientName = (String)getSetting(info, ClickHouseConnectionSettings.CLIENT_NAME); - this.useNewParser = (Boolean)getSetting(info, ClickHouseConnectionSettings.USE_NEW_PARSER); this.maxParallelReplicas = getSetting(info, ClickHouseQueryParam.MAX_PARALLEL_REPLICAS); this.maxPartitionsPerInsertBlock = getSetting(info, ClickHouseQueryParam.MAX_PARTITIONS_PER_INSERT_BLOCK); @@ -182,7 +179,6 @@ public Properties asProperties() { ret.put(ClickHouseConnectionSettings.SOCKET_TIMEOUT.getKey(), String.valueOf(socketTimeout)); ret.put(ClickHouseConnectionSettings.CONNECTION_TIMEOUT.getKey(), String.valueOf(connectionTimeout)); ret.put(ClickHouseConnectionSettings.DATA_TRANSFER_TIMEOUT.getKey(), String.valueOf(dataTransferTimeout)); - ret.put(ClickHouseConnectionSettings.KEEP_ALIVE_TIMEOUT.getKey(), String.valueOf(keepAliveTimeout)); ret.put(ClickHouseConnectionSettings.TIME_TO_LIVE_MILLIS.getKey(), String.valueOf(timeToLiveMillis)); ret.put(ClickHouseConnectionSettings.DEFAULT_MAX_PER_ROUTE.getKey(), String.valueOf(defaultMaxPerRoute)); ret.put(ClickHouseConnectionSettings.MAX_TOTAL.getKey(), String.valueOf(maxTotal)); @@ -199,9 +195,9 @@ public Properties asProperties() { ret.put(ClickHouseConnectionSettings.USE_TIME_ZONE.getKey(), String.valueOf(useTimeZone)); ret.put(ClickHouseConnectionSettings.USE_SERVER_TIME_ZONE_FOR_DATES.getKey(), String.valueOf(useServerTimeZoneForDates)); ret.put(ClickHouseConnectionSettings.USE_OBJECTS_IN_ARRAYS.getKey(), String.valueOf(useObjectsInArrays)); + ret.put(ClickHouseConnectionSettings.USE_SHARED_COOKIE_STORE.getKey(), String.valueOf(useSharedCookieStore)); ret.put(ClickHouseConnectionSettings.CLIENT_NAME.getKey(), String.valueOf(clientName)); - ret.put(ClickHouseConnectionSettings.USE_NEW_PARSER.getKey(), String.valueOf(useNewParser)); - + ret.put(ClickHouseQueryParam.MAX_PARALLEL_REPLICAS.getKey(), maxParallelReplicas); ret.put(ClickHouseQueryParam.MAX_PARTITIONS_PER_INSERT_BLOCK.getKey(), maxPartitionsPerInsertBlock); ret.put(ClickHouseQueryParam.TOTALS_MODE.getKey(), totalsMode); @@ -254,7 +250,6 @@ public ClickHouseProperties(ClickHouseProperties properties) { setSocketTimeout(properties.socketTimeout); setConnectionTimeout(properties.connectionTimeout); setDataTransferTimeout(properties.dataTransferTimeout); - setKeepAliveTimeout(properties.keepAliveTimeout); setTimeToLiveMillis(properties.timeToLiveMillis); setDefaultMaxPerRoute(properties.defaultMaxPerRoute); setMaxTotal(properties.maxTotal); @@ -271,8 +266,8 @@ public ClickHouseProperties(ClickHouseProperties properties) { setUseTimeZone(properties.useTimeZone); setUseServerTimeZoneForDates(properties.useServerTimeZoneForDates); setUseObjectsInArrays(properties.useObjectsInArrays); + setUseSharedCookieStore(properties.useSharedCookieStore); setClientName(properties.clientName); - setUseNewParser(properties.useNewParser); setMaxParallelReplicas(properties.maxParallelReplicas); setMaxPartitionsPerInsertBlock(properties.maxPartitionsPerInsertBlock); setTotalsMode(properties.totalsMode); @@ -567,16 +562,6 @@ public void setDataTransferTimeout(int dataTransferTimeout) { this.dataTransferTimeout = dataTransferTimeout; } - @Deprecated - public int getKeepAliveTimeout() { - return keepAliveTimeout; - } - - @Deprecated - public void setKeepAliveTimeout(int keepAliveTimeout) { - this.keepAliveTimeout = keepAliveTimeout; - } - public String getUser() { return user; } @@ -688,22 +673,20 @@ public void setUseObjectsInArrays(boolean useObjectsInArrays) { this.useObjectsInArrays = useObjectsInArrays; } - public String getClientName() { - return this.clientName; + public boolean isUseSharedCookieStore() { + return useSharedCookieStore; } - public void setClientName(String clientName) { - this.clientName = clientName; + public void setUseSharedCookieStore(boolean useSharedCookieStore) { + this.useSharedCookieStore = useSharedCookieStore; } - @Deprecated - public boolean isUseNewParser() { - return useNewParser; + public String getClientName() { + return this.clientName; } - @Deprecated - public void setUseNewParser(boolean useNewParser) { - this.useNewParser = useNewParser; + public void setClientName(String clientName) { + this.clientName = clientName; } public boolean isUseServerTimeZoneForDates() { diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java index 4d6e57baa..f33564093 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java @@ -11,6 +11,10 @@ public enum ClickHouseQueryParam implements DriverPropertyCreator { AGGREGATION_MEMORY_EFFICIENT_MERGE_THREADS("aggregation_memory_efficient_merge_threads", null, Long.class, ""), + ALLOW_EXPERIMENTAL_BIGINT_TYPES("allow_experimental_bigint_types", null, Integer.class, "Enables or disables integer values exceeding the range that is supported by the int data type."), + + ALLOW_EXPERIMENTAL_MAP_TYPE("allow_experimental_map_type", null, Integer.class, "Enables or disables Map data type."), + BACKGROUND_POOL_SIZE("background_pool_size", null, Long.class, ""), AUTHORIZATION("authorization", null, String.class, "Authorization header content for HTTP transport"), @@ -78,6 +82,8 @@ public enum ClickHouseQueryParam implements DriverPropertyCreator { INTERACTIVE_DELAY("interactive_delay", null, Long.class, ""), + JOIN_ALGORITHM("join_algorithm", null, String.class, ""), + LOAD_BALANCING("load_balancing", null, String.class, ""), LOG_QUERIES("log_queries", false, Boolean.class, ""), @@ -245,9 +251,9 @@ public enum ClickHouseQueryParam implements DriverPropertyCreator { QUOTA_KEY("quota_key", null, String.class, "quota is calculated for each quota_key value. For example here may be some user name."), + @Deprecated use_client_time_zone("use_client_time_zone", false, Boolean.class, ""), - USE_UNCOMPRESSED_CACHE("use_uncompressed_cache", true, Boolean.class, "Whether to use the cache of uncompressed blocks."), USER("user", null, String.class, "user name, by default - default"), @@ -261,7 +267,7 @@ public enum ClickHouseQueryParam implements DriverPropertyCreator { private final String key; private final Object defaultValue; - private final Class clazz; + private final Class clazz; private final String description; ClickHouseQueryParam(String key, T defaultValue, Class clazz, String description) { @@ -279,7 +285,7 @@ public Object getDefaultValue() { return defaultValue; } - public Class getClazz() { + public Class getClazz() { return clazz; } diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseCookieStoreProvider.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseCookieStoreProvider.java new file mode 100644 index 000000000..921226f66 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseCookieStoreProvider.java @@ -0,0 +1,29 @@ +package ru.yandex.clickhouse.util; + +import org.apache.http.client.CookieStore; +import org.apache.http.impl.client.BasicCookieStore; +import ru.yandex.clickhouse.settings.ClickHouseProperties; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class ClickHouseCookieStoreProvider { + private static final Map cookieStoreMap = new ConcurrentHashMap<>(); + + public CookieStore getCookieStore(ClickHouseProperties properties) { + return hasValidProperties(properties) && properties.isUseSharedCookieStore() ? + cookieStoreMap.computeIfAbsent(getCookieStoreKey(properties), k -> new BasicCookieStore()) : + null; + } + + private boolean hasValidProperties(ClickHouseProperties properties) { + return properties != null + && !Utils.isNullOrEmptyString(properties.getHost()) + && properties.getPort() > 0 + && !Utils.isNullOrEmptyString(properties.getDatabase()); + } + + private String getCookieStoreKey(ClickHouseProperties properties) { + return String.format("%s:%s/%s", properties.getHost(), properties.getPort(), properties.getDatabase()); + } +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseHttpClientBuilder.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseHttpClientBuilder.java index 9dbb8af6b..a7fd7956d 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseHttpClientBuilder.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseHttpClientBuilder.java @@ -36,6 +36,7 @@ import org.apache.http.client.AuthCache; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.config.ConnectionConfig; @@ -60,6 +61,7 @@ public class ClickHouseHttpClientBuilder { + private static final ClickHouseCookieStoreProvider cookieStoreProvider = new ClickHouseCookieStoreProvider(); private final ClickHouseProperties properties; public ClickHouseHttpClientBuilder(ClickHouseProperties properties) { @@ -76,6 +78,7 @@ public CloseableHttpClient buildClient() throws Exception { .setDefaultHeaders(getDefaultHeaders()) .setDefaultCredentialsProvider(getDefaultCredentialsProvider()) .disableContentCompression() // gzip is not needed. Use lz4 when compress=1 + .setDefaultCookieStore(cookieStoreProvider.getCookieStore(properties)) .disableRedirectHandling(); String clientName = properties != null ? properties.getClientName() : null; @@ -112,6 +115,7 @@ public static HttpClientContext createClientContext(ClickHouseProperties props) authCache.put(getTargetHost(props), basicAuth); HttpClientContext ctx = HttpClientContext.create(); ctx.setAuthCache(authCache); + ctx.setCookieStore(cookieStoreProvider.getCookieStore(props)); return ctx; } @@ -164,6 +168,7 @@ private RequestConfig getRequestConfig() { .setSocketTimeout(properties.getSocketTimeout()) .setConnectTimeout(properties.getConnectionTimeout()) .setConnectionRequestTimeout(properties.getConnectionTimeout()) + .setCookieSpec(CookieSpecs.STANDARD) .build(); } diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryInputStream.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryInputStream.java index 3fc97c263..5ba284902 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryInputStream.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryInputStream.java @@ -1,7 +1,6 @@ package ru.yandex.clickhouse.util; -import ru.yandex.clickhouse.domain.ClickHouseDataType; -import ru.yandex.clickhouse.settings.ClickHouseProperties; +import static ru.yandex.clickhouse.util.ClickHouseRowBinaryStream.MILLIS_IN_DAY; import java.io.Closeable; import java.io.DataInputStream; @@ -15,364 +14,404 @@ import java.nio.charset.StandardCharsets; import java.sql.Date; import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.TimeZone; import java.util.UUID; import java.util.concurrent.TimeUnit; - -import static ru.yandex.clickhouse.util.ClickHouseRowBinaryStream.MILLIS_IN_DAY; +import ru.yandex.clickhouse.domain.ClickHouseDataType; +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; +import ru.yandex.clickhouse.settings.ClickHouseProperties; public class ClickHouseRowBinaryInputStream implements Closeable { - private final DataInputStream in; - private final TimeZone timeZone; - - public ClickHouseRowBinaryInputStream(InputStream is, TimeZone timeZone, ClickHouseProperties properties) { - this.in = new DataInputStream(is); - if (properties.isUseServerTimeZoneForDates()) { - this.timeZone = timeZone; - } else { - this.timeZone = TimeZone.getDefault(); - } - } - - public void readBytes(byte[] bytes) throws IOException { - readBytes(bytes, 0, bytes.length); - } - - public void readBytes(byte[] bytes, int offset, int length) throws IOException { - while (length > 0) { - int read = in.read(bytes, offset, length); - if (read == -1) - throw new EOFException(); - offset += read; - length -= read; - } - } - - public int readByte() throws IOException { - return in.readUnsignedByte(); - } - - public boolean readIsNull() throws IOException { - int value = readByte(); - - Utils.checkArgument(value, 0, 1); - - return value != 0; - } - - public String readString() throws IOException { - int length = Utils.readUnsignedLeb128(in); - byte[] bytes = new byte[length]; - readBytes(bytes); - - return new String(bytes, StandardCharsets.UTF_8); - } - - public String readFixedString(int length) throws IOException { - byte[] bytes = new byte[length]; - readBytes(bytes); - - return new String(bytes, StandardCharsets.UTF_8); - } - - public boolean readBoolean() throws IOException { - int value = readUInt8(); - Utils.checkArgument(value, 0, 1); - return value != 0; - } - - public short readUInt8() throws IOException { - return (short) in.readUnsignedByte(); - } - - /** - * Warning: the result is negative in Java if UInt8 > 0x7f + private final DataInputStream in; + private final TimeZone timeZone; + + private final List columns; + + public ClickHouseRowBinaryInputStream(InputStream is, TimeZone timeZone, ClickHouseProperties properties) + throws IOException { + this(is, timeZone, properties, false); + } + + public ClickHouseRowBinaryInputStream(InputStream is, TimeZone timeZone, ClickHouseProperties properties, + boolean hasColumnInfo) throws IOException { + this.in = new DataInputStream(is); + if (properties.isUseServerTimeZoneForDates()) { + this.timeZone = timeZone; + } else { + this.timeZone = TimeZone.getDefault(); + } + + if (hasColumnInfo) { + // read names and types + int count = Utils.readVarInt(in); + String[][] columns = new String[count][2]; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < count; j++) { + columns[j][i] = readString(); + } + } + + List list = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + String[] column = columns[i]; + list.add(ClickHouseColumnInfo.parse(column[1], column[0], timeZone)); + } + this.columns = Collections.unmodifiableList(list); + } else { + this.columns = Collections.emptyList(); + } + } + + public List getColumns() { + return this.columns; + } + + public void readBytes(byte[] bytes) throws IOException { + readBytes(bytes, 0, bytes.length); + } + + public void readBytes(byte[] bytes, int offset, int length) throws IOException { + while (length > 0) { + int read = in.read(bytes, offset, length); + if (read == -1) { + throw new EOFException(); + } + offset += read; + length -= read; + } + } + + public int readByte() throws IOException { + return in.readUnsignedByte(); + } + + public boolean readIsNull() throws IOException { + int value = readByte(); + + Utils.checkArgument(value, 0, 1); + + return value != 0; + } + + public String readString() throws IOException { + int length = Utils.readUnsignedLeb128(in); + byte[] bytes = new byte[length]; + readBytes(bytes); + + return new String(bytes, StandardCharsets.UTF_8); + } + + public String readFixedString(int length) throws IOException { + byte[] bytes = new byte[length]; + readBytes(bytes); + + return new String(bytes, StandardCharsets.UTF_8); + } + + public boolean readBoolean() throws IOException { + int value = readUInt8(); + Utils.checkArgument(value, 0, 1); + return value != 0; + } + + public short readUInt8() throws IOException { + return (short) in.readUnsignedByte(); + } + + /** + * Warning: the result is negative in Java if UInt8 > 0x7f * - * @return next UInt8 value as a byte - * @throws IOException in case if an I/O error occurs - */ - public byte readUInt8AsByte() throws IOException { - return in.readByte(); - } - - public byte readInt8() throws IOException { - return in.readByte(); - } - - public int readUInt16() throws IOException { - return Utils.readUnsignedShort(in); - } - - /** - * Warning: the result is negative in Java if UInt16 > 0x7fff - * @return next UInt16 value as a short - * @throws IOException in case if an I/O error occurs - */ - public short readUInt16AsShort() throws IOException { - return (short) Utils.readUnsignedShort(in); - } - - public short readInt16() throws IOException { - return (short) Utils.readUnsignedShort(in); - } - - public long readUInt32() throws IOException { - return ((long) Utils.readInt(in)) & 0xffffffffL; - } - - /** - * Warning: the result is negative in Java if UInt32 > 0x7fffffff - * @return next UInt32 value as an int - * @throws IOException in case if an I/O error occurs - */ - public int readUInt32AsInt() throws IOException { - return Utils.readInt(in); - } - - public int readInt32() throws IOException { - return Utils.readInt(in); - } - - /** - * Warning: the result is negative in Java if UInt64 > 0x7fffffffffffffff - * @return next UInt64 value as a long - * @throws IOException in case if an I/O error occurs - */ - public long readUInt64AsLong() throws IOException { - return Utils.readLong(in); - } - - public BigInteger readUInt64() throws IOException { - return Utils.readLongAsBigInteger(in); - } - - public long readInt64() throws IOException { - return Utils.readLong(in); - } - - public BigInteger readInt128() throws IOException { + * @return next UInt8 value as a byte + * @throws IOException in case if an I/O error occurs + */ + public byte readUInt8AsByte() throws IOException { + return in.readByte(); + } + + public byte readInt8() throws IOException { + return in.readByte(); + } + + public int readUInt16() throws IOException { + return Utils.readUnsignedShort(in); + } + + /** + * Warning: the result is negative in Java if UInt16 > 0x7fff + * + * @return next UInt16 value as a short + * @throws IOException in case if an I/O error occurs + */ + public short readUInt16AsShort() throws IOException { + return (short) Utils.readUnsignedShort(in); + } + + public short readInt16() throws IOException { + return (short) Utils.readUnsignedShort(in); + } + + public long readUInt32() throws IOException { + return ((long) Utils.readInt(in)) & 0xffffffffL; + } + + /** + * Warning: the result is negative in Java if UInt32 > 0x7fffffff + * + * @return next UInt32 value as an int + * @throws IOException in case if an I/O error occurs + */ + public int readUInt32AsInt() throws IOException { + return Utils.readInt(in); + } + + public int readInt32() throws IOException { + return Utils.readInt(in); + } + + /** + * Warning: the result is negative in Java if UInt64 > 0x7fffffffffffffff + * + * @return next UInt64 value as a long + * @throws IOException in case if an I/O error occurs + */ + public long readUInt64AsLong() throws IOException { + return Utils.readLong(in); + } + + public BigInteger readUInt64() throws IOException { + return Utils.readLongAsBigInteger(in); + } + + public long readInt64() throws IOException { + return Utils.readLong(in); + } + + public BigInteger readInt128() throws IOException { return Utils.readBigInteger(in, 16); } public BigInteger writeUInt128() throws IOException { - return Utils.readBigInteger(in, 16); + return Utils.readBigInteger(in, 16); } public BigInteger writeInt256() throws IOException { - return Utils.readBigInteger(in, 32); + return Utils.readBigInteger(in, 32); } public BigInteger writeUInt256() throws IOException { return Utils.readBigInteger(in, 32); } - public Timestamp readDateTime() throws IOException { - long value = readUInt32(); - return new Timestamp(TimeUnit.SECONDS.toMillis(value)); - } - - public Date readDate() throws IOException { - int daysSinceEpoch = readUInt16(); - long utcMillis = daysSinceEpoch * MILLIS_IN_DAY; - long localMillis = utcMillis - timeZone.getOffset(utcMillis); - return new Date(localMillis); - } - - public float readFloat32() throws IOException { - return Float.intBitsToFloat(Utils.readInt(in)); - } - - public double readFloat64() throws IOException { - return Double.longBitsToDouble(Utils.readLong(in)); - } - - public Date[] readDateArray() throws IOException { - int length = Utils.readUnsignedLeb128(in); - Date[] dates = new Date[length]; - for (int i = 0; i < length; i++) { - dates[i] = readDate(); - } - - return dates; - } - - public Timestamp[] readDateTimeArray() throws IOException { - int length = Utils.readUnsignedLeb128(in); - Timestamp[] dates = new Timestamp[length]; - for (int i = 0; i < length; i++) { - dates[i] = readDateTime(); - } - - return dates; - } - - public String[] readStringArray() throws IOException { - int length = Utils.readUnsignedLeb128(in); - String[] strings = new String[length]; - for (int i = 0; i < length; i++) { - strings[i] = readString(); - } - - return strings; - } - - public byte[] readInt8Array() throws IOException { - int length = Utils.readUnsignedLeb128(in); - byte[] bytes = new byte[length]; - for (int i = 0; i < length; i++) { - bytes[i] = readInt8(); - } - - return bytes; - } - - public byte[] readUInt8ArrayAsByte() throws IOException { - int length = Utils.readUnsignedLeb128(in); - byte[] bytes = new byte[length]; - for (int i = 0; i < length; i++) { - bytes[i] = readUInt8AsByte(); - } - - return bytes; - } - - public short[] readUInt8Array() throws IOException { - int length = Utils.readUnsignedLeb128(in); - short[] shorts = new short[length]; - for (int i = 0; i < length; i++) { - shorts[i] = readUInt8(); - } - - return shorts; - } - - public short[] readInt16Array() throws IOException { - int length = Utils.readUnsignedLeb128(in); - short[] shorts = new short[length]; - for (int i = 0; i < length; i++) { - shorts[i] = readInt16(); - } - - return shorts; - } - - public short[] readUInt16ArrayAsShort() throws IOException { - int length = Utils.readUnsignedLeb128(in); - short[] shorts = new short[length]; - for (int i = 0; i < length; i++) { - shorts[i] = readUInt16AsShort(); - } - - return shorts; - } - - public int[] readUInt16Array() throws IOException { - int length = Utils.readUnsignedLeb128(in); - int[] ints = new int[length]; - for (int i = 0; i < length; i++) { - ints[i] = readUInt16(); - } - - return ints; - } - - public int[] readInt32Array() throws IOException { - int length = Utils.readUnsignedLeb128(in); - int[] ints = new int[length]; - for (int i = 0; i < length; i++) { - ints[i] = readInt32(); - } - - return ints; - } - - public int[] readUInt32ArrayAsInt() throws IOException { - int length = Utils.readUnsignedLeb128(in); - int[] ints = new int[length]; - for (int i = 0; i < length; i++) { - ints[i] = readUInt32AsInt(); - } - - return ints; - } - - public long[] readUInt32Array() throws IOException { - int length = Utils.readUnsignedLeb128(in); - long[] longs = new long[length]; - for (int i = 0; i < length; i++) { - longs[i] = readUInt32(); - } - - return longs; - } - - public long[] readInt64Array() throws IOException { - int length = Utils.readUnsignedLeb128(in); - long[] longs = new long[length]; - for (int i = 0; i < length; i++) { - longs[i] = readInt64(); - } - - return longs; - } - - public long[] readUInt64ArrayAsLong() throws IOException { - int length = Utils.readUnsignedLeb128(in); - long[] longs = new long[length]; - for (int i = 0; i < length; i++) { - longs[i] = readUInt64AsLong(); - } - - return longs; - } - - public BigInteger[] readUInt64Array() throws IOException { - int length = Utils.readUnsignedLeb128(in); - BigInteger[] bigs = new BigInteger[length]; - for (int i = 0; i < length; i++) { - bigs[i] = readUInt64(); - } - - return bigs; - } - - public float[] readFloat32Array() throws IOException { - int length = Utils.readUnsignedLeb128(in); - float[] floats = new float[length]; - for (int i = 0; i < length; i++) { - floats[i] = readFloat32(); - } - - return floats; - } - - public double[] readFloat64Array() throws IOException { - int length = Utils.readUnsignedLeb128(in); - double[] doubles = new double[length]; - for (int i = 0; i < length; i++) { - doubles[i] = readFloat64(); - } - - return doubles; - } - - public UUID readUUID() throws IOException { - byte[] array = new byte[16]; - readBytes(array); - - ByteBuffer bb = ByteBuffer.wrap(array).order(ByteOrder.LITTLE_ENDIAN); - return new UUID(bb.getLong(), bb.getLong()); - } - - public UUID[] readUUIDArray() throws IOException { - int length = Utils.readUnsignedLeb128(in); - UUID[] uuids = new UUID[length]; - for (int i = 0; i < length; i++) { - uuids[i] = readUUID(); - } - - return uuids; - } + public Timestamp readDateTime() throws IOException { + long value = readUInt32(); + return new Timestamp(TimeUnit.SECONDS.toMillis(value)); + } + + public Date readDate() throws IOException { + int daysSinceEpoch = readUInt16(); + long utcMillis = daysSinceEpoch * MILLIS_IN_DAY; + long localMillis = utcMillis - timeZone.getOffset(utcMillis); + return new Date(localMillis); + } + + public float readFloat32() throws IOException { + return Float.intBitsToFloat(Utils.readInt(in)); + } + + public double readFloat64() throws IOException { + return Double.longBitsToDouble(Utils.readLong(in)); + } + + public Date[] readDateArray() throws IOException { + int length = Utils.readUnsignedLeb128(in); + Date[] dates = new Date[length]; + for (int i = 0; i < length; i++) { + dates[i] = readDate(); + } + + return dates; + } + + public Timestamp[] readDateTimeArray() throws IOException { + int length = Utils.readUnsignedLeb128(in); + Timestamp[] dates = new Timestamp[length]; + for (int i = 0; i < length; i++) { + dates[i] = readDateTime(); + } + + return dates; + } + + public String[] readStringArray() throws IOException { + int length = Utils.readUnsignedLeb128(in); + String[] strings = new String[length]; + for (int i = 0; i < length; i++) { + strings[i] = readString(); + } + + return strings; + } + + public byte[] readInt8Array() throws IOException { + int length = Utils.readUnsignedLeb128(in); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = readInt8(); + } + + return bytes; + } + + public byte[] readUInt8ArrayAsByte() throws IOException { + int length = Utils.readUnsignedLeb128(in); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = readUInt8AsByte(); + } + + return bytes; + } + + public short[] readUInt8Array() throws IOException { + int length = Utils.readUnsignedLeb128(in); + short[] shorts = new short[length]; + for (int i = 0; i < length; i++) { + shorts[i] = readUInt8(); + } + + return shorts; + } + + public short[] readInt16Array() throws IOException { + int length = Utils.readUnsignedLeb128(in); + short[] shorts = new short[length]; + for (int i = 0; i < length; i++) { + shorts[i] = readInt16(); + } + + return shorts; + } + + public short[] readUInt16ArrayAsShort() throws IOException { + int length = Utils.readUnsignedLeb128(in); + short[] shorts = new short[length]; + for (int i = 0; i < length; i++) { + shorts[i] = readUInt16AsShort(); + } + + return shorts; + } + + public int[] readUInt16Array() throws IOException { + int length = Utils.readUnsignedLeb128(in); + int[] ints = new int[length]; + for (int i = 0; i < length; i++) { + ints[i] = readUInt16(); + } + + return ints; + } + + public int[] readInt32Array() throws IOException { + int length = Utils.readUnsignedLeb128(in); + int[] ints = new int[length]; + for (int i = 0; i < length; i++) { + ints[i] = readInt32(); + } + + return ints; + } + + public int[] readUInt32ArrayAsInt() throws IOException { + int length = Utils.readUnsignedLeb128(in); + int[] ints = new int[length]; + for (int i = 0; i < length; i++) { + ints[i] = readUInt32AsInt(); + } + + return ints; + } + + public long[] readUInt32Array() throws IOException { + int length = Utils.readUnsignedLeb128(in); + long[] longs = new long[length]; + for (int i = 0; i < length; i++) { + longs[i] = readUInt32(); + } + + return longs; + } + + public long[] readInt64Array() throws IOException { + int length = Utils.readUnsignedLeb128(in); + long[] longs = new long[length]; + for (int i = 0; i < length; i++) { + longs[i] = readInt64(); + } + + return longs; + } + + public long[] readUInt64ArrayAsLong() throws IOException { + int length = Utils.readUnsignedLeb128(in); + long[] longs = new long[length]; + for (int i = 0; i < length; i++) { + longs[i] = readUInt64AsLong(); + } + + return longs; + } + + public BigInteger[] readUInt64Array() throws IOException { + int length = Utils.readUnsignedLeb128(in); + BigInteger[] bigs = new BigInteger[length]; + for (int i = 0; i < length; i++) { + bigs[i] = readUInt64(); + } + + return bigs; + } + + public float[] readFloat32Array() throws IOException { + int length = Utils.readUnsignedLeb128(in); + float[] floats = new float[length]; + for (int i = 0; i < length; i++) { + floats[i] = readFloat32(); + } + + return floats; + } + + public double[] readFloat64Array() throws IOException { + int length = Utils.readUnsignedLeb128(in); + double[] doubles = new double[length]; + for (int i = 0; i < length; i++) { + doubles[i] = readFloat64(); + } + + return doubles; + } + + public UUID readUUID() throws IOException { + byte[] array = new byte[16]; + readBytes(array); + + ByteBuffer bb = ByteBuffer.wrap(array).order(ByteOrder.LITTLE_ENDIAN); + return new UUID(bb.getLong(), bb.getLong()); + } + + public UUID[] readUUIDArray() throws IOException { + int length = Utils.readUnsignedLeb128(in); + UUID[] uuids = new UUID[length]; + for (int i = 0; i < length; i++) { + uuids[i] = readUUID(); + } + + return uuids; + } public BigDecimal readDecimal32(int scale) throws IOException { int i = Utils.readInt(in); @@ -397,7 +436,7 @@ public BigDecimal readDecimal128(int scale) throws IOException { return res; } - public BigDecimal readDecimal256(int scale) throws IOException { + public BigDecimal readDecimal256(int scale) throws IOException { byte[] r = new byte[32]; for (int i = r.length; i > 0; i--) { r[i - 1] = (byte) in.readUnsignedByte(); @@ -406,12 +445,12 @@ public BigDecimal readDecimal256(int scale) throws IOException { return res; } - public ClickHouseBitmap readBitmap(ClickHouseDataType innerType) throws IOException { - return ClickHouseBitmap.deserialize(in, innerType); - } + public ClickHouseBitmap readBitmap(ClickHouseDataType innerType) throws IOException { + return ClickHouseBitmap.deserialize(in, innerType); + } - @Override - public void close() throws IOException { - in.close(); - } + @Override + public void close() throws IOException { + in.close(); + } } diff --git a/clickhouse-jdbc/src/main/javacc/ClickHouseSqlParser.jj b/clickhouse-jdbc/src/main/javacc/ClickHouseSqlParser.jj index 0810dfecd..fbdd709ff 100644 --- a/clickhouse-jdbc/src/main/javacc/ClickHouseSqlParser.jj +++ b/clickhouse-jdbc/src/main/javacc/ClickHouseSqlParser.jj @@ -70,40 +70,6 @@ public class ClickHouseSqlParser { return !(getToken(1).kind == AND && token_source.parentToken == BETWEEN); } - /** - * Parse given SQL. - * - * @deprecated This method will be removed in the near future. - *

- * Use {@link #parse(String, ClickHouseProperties)} instead. - * - * @param sql SQL query - * @param properties properties - * @return parsed SQL statement - */ - public static ClickHouseSqlStatement parseSingleStatement(String sql, ClickHouseProperties properties) { - return parseSingleStatement(sql, properties, null); - } - - /** - * Parse given SQL. - * - * @deprecated This method will be removed in the near future. - *

- * Use {@link #parse(String, ClickHouseProperties, ParseHandler)} instead. - * - * @param sql SQL query - * @param properties properties - * @param handler parse handler - * @return parsed SQL statement - */ - public static ClickHouseSqlStatement parseSingleStatement( - String sql, ClickHouseProperties properties, ParseHandler handler) { - ClickHouseSqlStatement[] stmts = parse(sql, properties, handler); - - return stmts.length == 1 ? stmts[0] : new ClickHouseSqlStatement(sql, StatementType.UNKNOWN); - } - public static ClickHouseSqlStatement[] parse(String sql, ClickHouseProperties properties) { return parse(sql, properties, null); } @@ -116,7 +82,7 @@ public class ClickHouseSqlParser { ClickHouseSqlStatement[] stmts = new ClickHouseSqlStatement[] { new ClickHouseSqlStatement(sql, StatementType.UNKNOWN) }; - if (!properties.isUseNewParser() || sql == null || sql.isEmpty()) { + if (sql == null || sql.isEmpty()) { return stmts; } @@ -127,7 +93,7 @@ public class ClickHouseSqlParser { if (DEBUG) { throw new IllegalArgumentException(e); } else { - log.warn("Failed to parse the given SQL. If you believe the SQL is valid, please feel free to open an issue on Github with the following SQL and exception attached.\n{}", sql, e); + log.warn("{}. If you believe the SQL is valid, please feel free to open an issue on Github with this warning and the following SQL attached.\n{}", e.getMessage(), sql); } } @@ -413,7 +379,7 @@ void deleteStmt(): {} { // https://clickhouse.tech/docs/en/sql-reference/statements/describe-table/ void describeStmt(): {} { ( | ) { token_source.table = "columns"; } - (LOOKAHEAD({ getToken(1).kind == TABLE }) )? (LOOKAHEAD(2) tableIdentifier(true) | anyExprList()) + (LOOKAHEAD({ getToken(1).kind == TABLE })
)? tableIdentifier(true) (anyExprList())? } // https://clickhouse.tech/docs/en/sql-reference/statements/detach/ @@ -713,6 +679,7 @@ void anyExpr(): {} { anyNestedExpr() ( LOOKAHEAD(2) ( + // TODO needs to extract parameters(and exclude ternary operator) here | | | | | operator() )? anyNestedExpr() )* diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/ClickHouseStatementTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/ClickHouseStatementTest.java index c5fadca00..bc9058fde 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/ClickHouseStatementTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/ClickHouseStatementTest.java @@ -11,6 +11,7 @@ import org.apache.http.impl.client.HttpClientBuilder; import org.testng.annotations.Test; +import ru.yandex.clickhouse.domain.ClickHouseFormat; import ru.yandex.clickhouse.settings.ClickHouseProperties; import ru.yandex.clickhouse.settings.ClickHouseQueryParam; @@ -22,41 +23,54 @@ public class ClickHouseStatementTest { @Test public void testClickhousify() throws Exception { + ClickHouseStatementImpl s = new ClickHouseStatementImpl(null, null, null, ResultSet.TYPE_FORWARD_ONLY); String sql = "SELECT ololo FROM ololoed;"; - assertEquals(ClickHouseStatementImpl.clickhousifySql(sql), "SELECT ololo FROM ololoed\nFORMAT TabSeparatedWithNamesAndTypes;"); + assertEquals(s.parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, null).getSQL(), + "SELECT ololo FROM ololoed\nFORMAT TabSeparatedWithNamesAndTypes"); - String sql2 = "SELECT ololo FROM ololoed"; - assertEquals(ClickHouseStatementImpl.clickhousifySql(sql2), "SELECT ololo FROM ololoed\nFORMAT TabSeparatedWithNamesAndTypes;"); + sql = "SELECT ololo FROM ololoed"; + assertEquals(s.parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, null).getSQL(), + "SELECT ololo FROM ololoed\nFORMAT TabSeparatedWithNamesAndTypes"); - String sql3 = "SELECT ololo FROM ololoed FORMAT TabSeparatedWithNamesAndTypes"; - assertEquals(ClickHouseStatementImpl.clickhousifySql(sql3), "SELECT ololo FROM ololoed FORMAT TabSeparatedWithNamesAndTypes"); + sql = "SELECT ololo FROM ololoed FORMAT TabSeparatedWithNamesAndTypes"; + assertEquals(s.parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, null).getSQL(), + "SELECT ololo FROM ololoed FORMAT TabSeparatedWithNamesAndTypes"); - String sql4 = "SELECT ololo FROM ololoed FORMAT TabSeparatedWithNamesAndTypes;"; - assertEquals(ClickHouseStatementImpl.clickhousifySql(sql4), "SELECT ololo FROM ololoed FORMAT TabSeparatedWithNamesAndTypes;"); + sql = "SELECT ololo FROM ololoed FORMAT TabSeparatedWithNamesAndTypes;"; + assertEquals(s.parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, null).getSQL(), + "SELECT ololo FROM ololoed FORMAT TabSeparatedWithNamesAndTypes"); - String sql5 = "SHOW ololo FROM ololoed;"; - assertEquals(ClickHouseStatementImpl.clickhousifySql(sql5), "SHOW ololo FROM ololoed\nFORMAT TabSeparatedWithNamesAndTypes;"); + sql = "SHOW ololo FROM ololoed;"; + assertEquals(s.parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, null).getSQL(), + "SHOW ololo FROM ololoed\nFORMAT TabSeparatedWithNamesAndTypes"); - String sql6 = " show ololo FROM ololoed;"; - assertEquals(ClickHouseStatementImpl.clickhousifySql(sql6), "show ololo FROM ololoed\nFORMAT TabSeparatedWithNamesAndTypes;"); + sql = " show ololo FROM ololoed;"; + assertEquals(s.parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, null).getSQL(), + " show ololo FROM ololoed\nFORMAT TabSeparatedWithNamesAndTypes"); - String sql7 = "SELECT ololo FROM ololoed \nFORMAT TabSeparatedWithNamesAndTypes"; - assertEquals(ClickHouseStatementImpl.clickhousifySql(sql7), "SELECT ololo FROM ololoed \nFORMAT TabSeparatedWithNamesAndTypes"); + sql = "SELECT ololo FROM ololoed \nFORMAT TabSeparatedWithNamesAndTypes"; + assertEquals(s.parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, null).getSQL(), + "SELECT ololo FROM ololoed \nFORMAT TabSeparatedWithNamesAndTypes"); - String sql8 = "SELECT ololo FROM ololoed \n\n FORMAT TabSeparatedWithNamesAndTypes"; - assertEquals(ClickHouseStatementImpl.clickhousifySql(sql8), "SELECT ololo FROM ololoed \n\n FORMAT TabSeparatedWithNamesAndTypes"); + sql = "SELECT ololo FROM ololoed \n\n FORMAT TabSeparatedWithNamesAndTypes"; + assertEquals(s.parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, null).getSQL(), + "SELECT ololo FROM ololoed \n\n FORMAT TabSeparatedWithNamesAndTypes"); - String sql9 = "SELECT ololo FROM ololoed\n-- some comments one line"; - assertEquals(ClickHouseStatementImpl.clickhousifySql(sql9), "SELECT ololo FROM ololoed\n-- some comments one line\nFORMAT TabSeparatedWithNamesAndTypes;"); + sql = "SELECT ololo FROM ololoed\n-- some comments one line"; + assertEquals(s.parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, null).getSQL(), + "SELECT ololo FROM ololoed\n-- some comments one line\nFORMAT TabSeparatedWithNamesAndTypes"); - String sql10 = "SELECT ololo FROM ololoed\n-- some comments\ntwo line"; - assertEquals(ClickHouseStatementImpl.clickhousifySql(sql10), "SELECT ololo FROM ololoed\n-- some comments\ntwo line\nFORMAT TabSeparatedWithNamesAndTypes;"); + sql = "SELECT ololo FROM ololoed\n-- some comments\ntwo line"; + assertEquals(s.parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, null).getSQL(), + "SELECT ololo FROM ololoed\n-- some comments\ntwo line\nFORMAT TabSeparatedWithNamesAndTypes"); - String sql11 = "SELECT ololo FROM ololoed/*\nsome comments\ntwo line*/"; - assertEquals(ClickHouseStatementImpl.clickhousifySql(sql11), "SELECT ololo FROM ololoed/*\nsome comments\ntwo line*/\nFORMAT TabSeparatedWithNamesAndTypes;"); + sql = "SELECT ololo FROM ololoed/*\nsome comments\ntwo line*/"; + assertEquals(s.parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, null).getSQL(), + "SELECT ololo FROM ololoed/*\nsome comments\ntwo line*/\nFORMAT TabSeparatedWithNamesAndTypes"); - String sql12 = "SELECT ololo FROM ololoed\n// c style some comments one line"; - assertEquals(ClickHouseStatementImpl.clickhousifySql(sql12), "SELECT ololo FROM ololoed\n// c style some comments one line\nFORMAT TabSeparatedWithNamesAndTypes;"); + sql = "SELECT ololo FROM ololoed\n// c style some comments one line"; + assertEquals(s.parseSqlStatements(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes, null).getSQL(), + "SELECT ololo FROM ololoed\n// c style some comments one line\nFORMAT TabSeparatedWithNamesAndTypes"); } @@ -174,32 +188,33 @@ public void testAdditionalDBParams() { } @Test - public void testIsSelect() { - assertTrue(ClickHouseStatementImpl.isSelect("SELECT 42")); - assertTrue(ClickHouseStatementImpl.isSelect("select 42")); - assertFalse(ClickHouseStatementImpl.isSelect("selectfoo")); - assertTrue(ClickHouseStatementImpl.isSelect(" SELECT foo")); - assertTrue(ClickHouseStatementImpl.isSelect("WITH foo")); - assertTrue(ClickHouseStatementImpl.isSelect("DESC foo")); - assertTrue(ClickHouseStatementImpl.isSelect("EXISTS foo")); - assertTrue(ClickHouseStatementImpl.isSelect("SHOW foo")); - assertTrue(ClickHouseStatementImpl.isSelect("-- foo\n SELECT 42")); - assertTrue(ClickHouseStatementImpl.isSelect("--foo\n SELECT 42")); - assertFalse(ClickHouseStatementImpl.isSelect("- foo\n SELECT 42")); - assertTrue(ClickHouseStatementImpl.isSelect("/* foo */ SELECT 42")); - assertTrue(ClickHouseStatementImpl.isSelect("/*\n * foo\n*/\n SELECT 42")); - assertFalse(ClickHouseStatementImpl.isSelect("/ foo */ SELECT 42")); - assertFalse(ClickHouseStatementImpl.isSelect("-- SELECT baz\n UPDATE foo")); - assertFalse(ClickHouseStatementImpl.isSelect("/* SELECT baz */\n UPDATE foo")); - assertFalse(ClickHouseStatementImpl.isSelect("/*\n UPDATE foo")); - assertFalse(ClickHouseStatementImpl.isSelect("/*")); - assertFalse(ClickHouseStatementImpl.isSelect("/**/")); - assertFalse(ClickHouseStatementImpl.isSelect(" --")); - assertTrue(ClickHouseStatementImpl.isSelect("explain select 42")); - assertTrue(ClickHouseStatementImpl.isSelect("EXPLAIN select 42")); - assertFalse(ClickHouseStatementImpl.isSelect("--EXPLAIN select 42\n alter")); - assertTrue(ClickHouseStatementImpl.isSelect("--\nEXPLAIN select 42")); - assertTrue(ClickHouseStatementImpl.isSelect("/*test*/ EXPLAIN select 42")); + public void testIsSelect() throws SQLException { + ClickHouseStatementImpl s = new ClickHouseStatementImpl(null, null, null, ResultSet.TYPE_FORWARD_ONLY); + assertTrue(s.parseSqlStatements("SELECT 42")[0].isQuery()); + assertTrue(s.parseSqlStatements("select 42")[0].isQuery()); + assertFalse(s.parseSqlStatements("selectfoo")[0].isQuery()); + assertTrue(s.parseSqlStatements(" SELECT foo")[0].isQuery()); + assertFalse(s.parseSqlStatements("WITH foo")[0].isQuery()); + assertTrue(s.parseSqlStatements("DESC foo")[0].isQuery()); + assertTrue(s.parseSqlStatements("EXISTS foo")[0].isQuery()); + assertTrue(s.parseSqlStatements("SHOW foo")[0].isQuery()); + assertTrue(s.parseSqlStatements("-- foo\n SELECT 42")[0].isQuery()); + assertTrue(s.parseSqlStatements("--foo\n SELECT 42")[0].isQuery()); + assertFalse(s.parseSqlStatements("- foo\n SELECT 42")[0].isQuery()); + assertTrue(s.parseSqlStatements("/* foo */ SELECT 42")[0].isQuery()); + assertTrue(s.parseSqlStatements("/*\n * foo\n*/\n SELECT 42")[0].isQuery()); + assertFalse(s.parseSqlStatements("/ foo */ SELECT 42")[0].isQuery()); + assertFalse(s.parseSqlStatements("-- SELECT baz\n UPDATE foo")[0].isQuery()); + assertFalse(s.parseSqlStatements("/* SELECT baz */\n UPDATE foo")[0].isQuery()); + assertFalse(s.parseSqlStatements("/*\n UPDATE foo")[0].isQuery()); + assertFalse(s.parseSqlStatements("/*")[0].isQuery()); + assertFalse(s.parseSqlStatements("/**/")[0].isQuery()); + assertFalse(s.parseSqlStatements(" --")[0].isQuery()); + assertTrue(s.parseSqlStatements("explain select 42")[0].isQuery()); + assertTrue(s.parseSqlStatements("EXPLAIN select 42")[0].isQuery()); + assertFalse(s.parseSqlStatements("--EXPLAIN select 42\n alter")[0].isQuery()); + assertTrue(s.parseSqlStatements("--\nEXPLAIN select 42")[0].isQuery()); + assertTrue(s.parseSqlStatements("/*test*/ EXPLAIN select 42")[0].isQuery()); } } diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/BatchInsertsTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/BatchInsertsTest.java index beae7a255..2065d7295 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/BatchInsertsTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/BatchInsertsTest.java @@ -5,12 +5,15 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.sql.Timestamp; import java.sql.Types; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Collections; import java.util.TimeZone; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.testng.Assert; import org.testng.annotations.AfterTest; @@ -289,4 +292,28 @@ public void testNullParameters() throws SQLException { st.addBatch(); } + @Test + public void testBatchInsertWithLongQuery() throws SQLException { + int columnCount = 200; + try (Statement s = connection.createStatement()) { + String createColumns = IntStream.range(0, columnCount).mapToObj( + i -> "`looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongnaaaaameeeeeeee" + i + "` String " + ).collect(Collectors.joining(",")); + s.execute("DROP TABLE IF EXISTS test.batch_insert_with_long_query"); + s.execute("CREATE TABLE test.batch_insert_with_long_query (" + createColumns + ") ENGINE = Memory"); + } + + String values = IntStream.range(0, columnCount).mapToObj(i -> "?").collect(Collectors.joining(",")); + String columns = IntStream.range(0, columnCount).mapToObj( + i -> "looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongnaaaaameeeeeeee" + i + ).collect(Collectors.joining(",")); + int index = 1; + try (PreparedStatement s = connection.prepareStatement("INSERT INTO test.batch_insert_with_long_query (" + columns + ") VALUES (" + values + ")")) { + for (int i = 0; i < columnCount; i++) { + s.setString(index++, "12345"); + } + s.addBatch(); + s.executeBatch(); + } + } } diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseConnectionImplTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseConnectionImplTest.java index 5a4d90676..99a7340c8 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseConnectionImplTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseConnectionImplTest.java @@ -1,15 +1,20 @@ package ru.yandex.clickhouse.integration; import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; import javax.sql.DataSource; import org.testng.annotations.Test; import ru.yandex.clickhouse.ClickHouseContainerForTest; +import ru.yandex.clickhouse.ClickHouseDataSource; import ru.yandex.clickhouse.except.ClickHouseException; import ru.yandex.clickhouse.settings.ClickHouseProperties; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseLargeNumberTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseLargeNumberTest.java index 7c972f410..e0eb722e3 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseLargeNumberTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseLargeNumberTest.java @@ -1,23 +1,32 @@ package ru.yandex.clickhouse.integration; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Connection; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; +import java.util.EnumMap; +import java.util.Map; import java.util.UUID; import org.testng.annotations.AfterTest; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import ru.yandex.clickhouse.ClickHouseConnection; import ru.yandex.clickhouse.ClickHouseContainerForTest; import ru.yandex.clickhouse.ClickHouseDataSource; +import ru.yandex.clickhouse.ClickHouseStatement; import ru.yandex.clickhouse.except.ClickHouseException; import ru.yandex.clickhouse.settings.ClickHouseProperties; +import ru.yandex.clickhouse.settings.ClickHouseQueryParam; public class ClickHouseLargeNumberTest { private Connection conn; @@ -46,6 +55,41 @@ public void tearDown() throws Exception { } } + @Test + public void testBigIntSupport() throws SQLException { + if (conn == null) { + return; + } + + String testSql = "create table if not exists system.test_bigint_support(i Int256) engine=Memory;" + + "drop table if exists system.test_bigint_support;"; + try (Connection conn = ClickHouseContainerForTest.newDataSource().getConnection(); + Statement s = conn.createStatement()) { + s.execute("set allow_experimental_bigint_types=0;" + testSql); + fail("Should fail without enabling bigint support"); + } catch (SQLException e) { + assertEquals(e.getErrorCode(), 44); + } + + try (Connection conn = ClickHouseContainerForTest.newDataSource().getConnection(); + Statement s = conn.createStatement()) { + assertFalse(s.execute("set allow_experimental_bigint_types=1;" + testSql)); + } + + try (ClickHouseConnection conn = ClickHouseContainerForTest.newDataSource().getConnection(); + ClickHouseStatement s = conn.createStatement()) { + Map params = new EnumMap<>(ClickHouseQueryParam.class); + params.put(ClickHouseQueryParam.ALLOW_EXPERIMENTAL_BIGINT_TYPES, "1"); + assertNull(s.executeQuery(testSql, params)); + + params.put(ClickHouseQueryParam.ALLOW_EXPERIMENTAL_BIGINT_TYPES, "0"); + s.executeQuery(testSql, params); + fail("Should fail without enabling bigint support"); + } catch (SQLException e) { + assertEquals(e.getErrorCode(), 44); + } + } + @Test public void testSignedIntegers() throws Exception { if (conn == null) { diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseMapTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseMapTest.java index 562793928..fe19bd933 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseMapTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseMapTest.java @@ -2,12 +2,17 @@ import static org.junit.Assert.assertArrayEquals; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; +import java.util.EnumMap; import java.util.Map; import java.util.UUID; @@ -15,14 +20,18 @@ import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import ru.yandex.clickhouse.ClickHouseConnection; import ru.yandex.clickhouse.ClickHouseContainerForTest; import ru.yandex.clickhouse.ClickHouseDataSource; +import ru.yandex.clickhouse.ClickHouseStatement; import ru.yandex.clickhouse.except.ClickHouseException; import ru.yandex.clickhouse.settings.ClickHouseProperties; +import ru.yandex.clickhouse.settings.ClickHouseQueryParam; +import ru.yandex.clickhouse.util.ClickHouseVersionNumberUtil; import ru.yandex.clickhouse.util.Utils; public class ClickHouseMapTest { - private Connection conn; + private ClickHouseConnection conn; @BeforeTest public void setUp() throws Exception { @@ -61,12 +70,54 @@ private void assertMap(Object actual, Object expected) { } } + @Test + public void testMapSupport() throws SQLException { + if (conn == null) { + return; + } + + String testSql = "create table if not exists system.test_map_support(m Map(UInt8, String)) engine=Memory;" + + "drop table if exists system.test_map_support;"; + try (Connection conn = ClickHouseContainerForTest.newDataSource().getConnection(); + Statement s = conn.createStatement()) { + s.execute("set allow_experimental_map_type=0;" + testSql); + fail("Should fail without enabling map support"); + } catch (SQLException e) { + assertEquals(e.getErrorCode(), 44); + } + + try (Connection conn = ClickHouseContainerForTest.newDataSource().getConnection(); + Statement s = conn.createStatement()) { + assertFalse(s.execute("set allow_experimental_map_type=1;" + testSql)); + } + + try (ClickHouseConnection conn = ClickHouseContainerForTest.newDataSource().getConnection(); + ClickHouseStatement s = conn.createStatement()) { + Map params = new EnumMap<>(ClickHouseQueryParam.class); + params.put(ClickHouseQueryParam.ALLOW_EXPERIMENTAL_MAP_TYPE, "1"); + assertNull(s.executeQuery(testSql, params)); + + params.put(ClickHouseQueryParam.ALLOW_EXPERIMENTAL_MAP_TYPE, "0"); + s.executeQuery(testSql, params); + fail("Should fail without enabling map support"); + } catch (SQLException e) { + assertEquals(e.getErrorCode(), 44); + } + } + @Test public void testMaps() throws Exception { if (conn == null) { return; } + // skip 21.4 + String serverVersion = conn.getServerVersion(); + if (ClickHouseVersionNumberUtil.getMajorVersion(serverVersion) == 21 + && ClickHouseVersionNumberUtil.getMinorVersion(serverVersion) == 4) { + return; + } + try (Statement s = conn.createStatement()) { s.execute("DROP TABLE IF EXISTS test_maps"); s.execute( diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHousePreparedStatementTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHousePreparedStatementTest.java index 3f283b8d7..c70dddf7f 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHousePreparedStatementTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHousePreparedStatementTest.java @@ -32,6 +32,10 @@ import ru.yandex.clickhouse.settings.ClickHouseProperties; import static java.util.Collections.singletonList; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; public class ClickHousePreparedStatementTest { @@ -676,6 +680,93 @@ public void testStaticNullValue() throws Exception { ps0.executeUpdate(); } + // known issue + public void testTernaryOperator() throws Exception { + String sql = "select x > 2 ? 'a' : 'b' from (select number as x from system.numbers limit ?)"; + try (PreparedStatement s = connection.prepareStatement(sql)) { + int len = 5; + s.setInt(1, len); + ResultSet rs = s.executeQuery(); + for (int i = 0; i < len; i++) { + assertTrue(rs.next()); + assertEquals(rs.getString(1), i > 2 ? "a" : "b"); + } + assertFalse(rs.next()); + rs.close(); + } + } + + @Test + public void testBatchProcess() throws Exception { + try (PreparedStatement s = connection.prepareStatement( + "create table if not exists test.batch_update(k UInt8, v String) engine=MergeTree order by k")) { + s.execute(); + } + + Object[][] data = new Object[][] { + new Object[] {1, "a"}, + new Object[] {1, "b"}, + new Object[] {3, "c"} + }; + + // insert + try (PreparedStatement s = connection.prepareStatement("insert into table test.batch_update values(?,?)")) { + for (int i = 0; i < data.length; i++) { + Object[] row = data[i]; + s.setInt(1, (int) row[0]); + s.setString(2, (String) row[1]); + s.addBatch(); + } + int[] results = s.executeBatch(); + assertNotNull(results); + assertEquals(results.length, 3); + } + + // select + try (PreparedStatement s = connection.prepareStatement( + "select * from test.batch_update where k in (?, ?) order by k, v")) { + s.setInt(1, 1); + s.setInt(2, 3); + ResultSet rs = s.executeQuery(); + int index = 0; + while (rs.next()) { + Object[] row = data[index++]; + assertEquals(rs.getInt(1), (int) row[0]); + assertEquals(rs.getString(2), (String) row[1]); + } + assertEquals(index, data.length); + } + + // update + try (PreparedStatement s = connection.prepareStatement( + "alter table test.batch_update update v = ? where k = ?")) { + s.setString(1, "x"); + s.setInt(2, 1); + s.addBatch(); + s.setString(1, "y"); + s.setInt(2, 3); + s.addBatch(); + int[] results = s.executeBatch(); + assertNotNull(results); + assertEquals(results.length, 2); + } + + // delete + try (PreparedStatement s = connection.prepareStatement("alter table test.batch_update delete where k = ?")) { + s.setInt(1, 1); + s.addBatch(); + s.setInt(1, 3); + s.addBatch(); + int[] results = s.executeBatch(); + assertNotNull(results); + assertEquals(results.length, 2); + } + + try (PreparedStatement s = connection.prepareStatement("drop table if exists test.batch_update")) { + s.execute(); + } + } + private static byte[] randomEncodedUUID() { UUID uuid = UUID.randomUUID(); return ByteBuffer.allocate(16) diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseStatementImplTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseStatementImplTest.java index 02ea533b1..c1235bff5 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseStatementImplTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseStatementImplTest.java @@ -1,9 +1,10 @@ package ru.yandex.clickhouse.integration; import static org.testng.Assert.assertEquals; -import static org.testng.AssertJUnit.assertNotNull; -import static org.testng.AssertJUnit.assertNull; -import static org.testng.AssertJUnit.assertTrue; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -34,6 +35,7 @@ import ru.yandex.clickhouse.ClickHouseStatement; import ru.yandex.clickhouse.settings.ClickHouseProperties; import ru.yandex.clickhouse.settings.ClickHouseQueryParam; +import ru.yandex.clickhouse.util.ClickHouseVersionNumberUtil; public class ClickHouseStatementImplTest { @@ -126,8 +128,10 @@ public void testSelectUInt64() throws SQLException { @Test public void testExternalData() throws SQLException, UnsupportedEncodingException { + String serverVersion = connection.getServerVersion(); ClickHouseStatement stmt = connection.createStatement(); - String[] rows = "21.3.3.14".equals(connection.getServerVersion()) + String[] rows = ClickHouseVersionNumberUtil.getMajorVersion(serverVersion) >= 21 + && ClickHouseVersionNumberUtil.getMinorVersion(serverVersion) >= 3 ? new String[] { "1\tGroup\n" } : new String[] { "1\tGroup", "1\tGroup\n" }; @@ -150,6 +154,41 @@ public void testExternalData() throws SQLException, UnsupportedEncodingException } } + // reproduce issue #634 + @Test + public void testLargeQueryWithExternalData() throws Exception { + String serverVersion = connection.getServerVersion(); + String[] rows = ClickHouseVersionNumberUtil.getMajorVersion(serverVersion) >= 21 + && ClickHouseVersionNumberUtil.getMinorVersion(serverVersion) >= 3 + ? new String[] { "1\tGroup\n" } + : new String[] { "1\tGroup", "1\tGroup\n" }; + + int length = 160000; + StringBuilder builder = new StringBuilder(length); + for (int i = 0; i < length; i++) { + builder.append('u'); + } + String user = builder.toString(); + for (String row : rows) { + try (ClickHouseStatement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery( + "select UserName, GroupName from (select '" + + user + + "' as UserName, 1 as GroupId) as g" + + "any left join groups using GroupId", null, + Collections.singletonList(new ClickHouseExternalData( + "groups", new ByteArrayInputStream(row.getBytes()) + ).withStructure("GroupId UInt8, GroupName String")))) { + Assert.assertTrue(rs.next()); + String userName = rs.getString("UserName"); + String groupName = rs.getString("GroupName"); + + Assert.assertEquals(userName, user); + Assert.assertEquals(groupName, "Group"); + } + } + } + private InputStream getTSVStream(final int rowsCount) { return new InputStream() { @@ -323,12 +362,12 @@ public void run() { assertNotNull( String.format("it's actually very strange. It seems the query hasn't been executed in %s seconds", timeout), queryId); - assertNull("An exception happened while the query was being executed", exceptionAtomicReference.get()); + assertNull(exceptionAtomicReference.get(), "An exception happened while the query was being executed"); - assertTrue("The query isn't being executed. It seems very strange", checkQuery(queryId, true,10)); + assertTrue(checkQuery(queryId, true, 10), "The query isn't being executed. It seems very strange"); firstStatement.cancel(); - assertTrue("The query is still being executed", checkQuery(queryId, false, 10)); + assertTrue(checkQuery(queryId, false, 10), "The query is still being executed"); firstStatement.close(); thread.interrupt(); @@ -359,14 +398,14 @@ public void run() { thread.setDaemon(true); thread.start(); final long timeout = 10; - assertTrue( - String.format("it's actually very strange. It seems the query hasn't been executed in %s seconds", timeout), - countDownLatch.await(timeout, TimeUnit.SECONDS)); - assertNull("An exception happened while the query was being executed", exceptionAtomicReference.get()); + assertTrue(countDownLatch.await(timeout, TimeUnit.SECONDS), + String.format( + "it's actually very strange. It seems the query hasn't been executed in %s seconds", timeout)); + assertNull(exceptionAtomicReference.get(), "An exception happened while the query was being executed"); - assertTrue("The query isn't being executed. It seems very strange", checkQuery(queryId, true,10)); + assertTrue(checkQuery(queryId, true, 10), "The query isn't being executed. It seems very strange"); firstStatement.cancel(); - assertTrue("The query is still being executed", checkQuery(queryId, false, 10)); + assertTrue(checkQuery(queryId, false, 10), "The query is still being executed"); firstStatement.close(); thread.interrupt(); @@ -406,6 +445,48 @@ public void testInsertQueryUUIDArray() throws SQLException { new UUID[] {UUID.fromString("5ff22319-793d-4e6c-bdc1-916095a5a496")}); } + @Test + public void testMultiStatements() throws SQLException { + try (Statement s = connection.createStatement()) { + String sql = "select 1; select 2"; + try (ResultSet rs = s.executeQuery(sql)) { + assertTrue(rs.next()); + assertEquals(rs.getString(1), "2"); + assertFalse(rs.next()); + } + + assertTrue(s.execute(sql)); + try (ResultSet rs = s.getResultSet()) { + assertNotNull(rs); + assertTrue(rs.next()); + assertEquals(rs.getString(1), "2"); + assertFalse(rs.next()); + } + + assertEquals(s.executeUpdate(sql), 1); + } + } + + @Test + public void testBatchProcessing() throws SQLException { + try (Statement s = connection.createStatement()) { + int[] results = s.executeBatch(); + assertNotNull(results); + assertEquals(results.length, 0); + + s.addBatch("select 1; select 2"); + s.addBatch("select 3"); + results = s.executeBatch(); + assertNotNull(results); + assertEquals(results.length, 3); + + s.clearBatch(); + results = s.executeBatch(); + assertNotNull(results); + assertEquals(results.length, 0); + } + } + private static Object readField(Object object, String fieldName, long timeoutSecs) { long start = System.currentTimeMillis(); Object value; diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlParserTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlParserTest.java new file mode 100644 index 000000000..e95ef61d1 --- /dev/null +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlParserTest.java @@ -0,0 +1,637 @@ +package ru.yandex.clickhouse.jdbc.parser; + +import org.testng.annotations.Test; + +import ru.yandex.clickhouse.settings.ClickHouseProperties; + +import static org.testng.Assert.assertEquals; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ClickHouseSqlParserTest { + private ClickHouseSqlStatement[] parse(String sql) { + return ClickHouseSqlParser.parse(sql, new ClickHouseProperties()); + } + + private String loadSql(String file) { + InputStream inputStream = ClickHouseSqlParserTest.class.getResourceAsStream("/sqls/" + file); + + StringBuilder sql = new StringBuilder(); + try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) { + String line; + while ((line = br.readLine()) != null) { + sql.append(line).append("\n"); + } + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + + return sql.toString(); + } + + private void checkSingleStatement(ClickHouseSqlStatement[] stmts, String sql) { + checkSingleStatement(stmts, sql, StatementType.UNKNOWN, ClickHouseSqlStatement.DEFAULT_DATABASE, + ClickHouseSqlStatement.DEFAULT_TABLE); + } + + private void checkSingleStatement(ClickHouseSqlStatement[] stmts, String sql, StatementType stmtType) { + checkSingleStatement(stmts, sql, stmtType, ClickHouseSqlStatement.DEFAULT_DATABASE, + ClickHouseSqlStatement.DEFAULT_TABLE); + } + + private void checkSingleStatement(ClickHouseSqlStatement[] stmts, String sql, StatementType stmtType, + String database, String table) { + assertEquals(stmts.length, 1); + + ClickHouseSqlStatement s = stmts[0]; + assertEquals(s.getSQL(), sql); + assertEquals(s.getStatementType(), stmtType); + assertEquals(s.getDatabaseOrDefault(null), database); + assertEquals(s.getTable(), table); + } + + @Test + public void testParseNonSql() throws ParseException { + String sql; + + assertEquals(parse(sql = null), + new ClickHouseSqlStatement[] { new ClickHouseSqlStatement(sql, StatementType.UNKNOWN) }); + assertEquals(parse(sql = ""), + new ClickHouseSqlStatement[] { new ClickHouseSqlStatement(sql, StatementType.UNKNOWN) }); + + checkSingleStatement(parse(sql = "invalid sql"), sql); + checkSingleStatement(parse(sql = "-- some comments"), sql); + checkSingleStatement(parse(sql = "/*********\r\n\r\t some ***** comments*/"), sql); + + checkSingleStatement(parse(sql = "select"), sql, StatementType.UNKNOWN); + checkSingleStatement(parse(sql = "select ()"), sql, StatementType.UNKNOWN); + checkSingleStatement(parse(sql = "select (()"), sql, StatementType.UNKNOWN); + checkSingleStatement(parse(sql = "select [[]"), sql, StatementType.UNKNOWN); + // checkSingleStatement(parse(sql = "select 1 select"), sql, + // StatementType.UNKNOWN); + } + + @Test + public void testAlterStatement() { + String sql; + + checkSingleStatement(parse(sql = "ALTER TABLE alter_test ADD COLUMN Added0 UInt32"), sql, StatementType.ALTER, + "system", "alter_test"); + checkSingleStatement( + parse(sql = "ALTER TABLE test_db.test_table UPDATE a = 1, \"b\" = '2', `c`=3.3 WHERE d=123 and e=456"), + sql, StatementType.ALTER_UPDATE, "test_db", "test_table"); + checkSingleStatement(parse(sql = "ALTER TABLE tTt on cluster 'cc' delete WHERE d=123 and e=456"), sql, + StatementType.ALTER_DELETE, "system", "tTt"); + checkSingleStatement(parse(sql = "ALTER USER user DEFAULT ROLE role1, role2"), sql, StatementType.ALTER); + } + + @Test + public void testAttachStatement() { + String sql; + + checkSingleStatement(parse(sql = "ATTACH TABLE IF NOT EXISTS t.t ON CLUSTER cluster"), sql, + StatementType.ATTACH); + } + + @Test + public void testCheckStatement() { + String sql; + + checkSingleStatement(parse(sql = "check table a"), sql, StatementType.CHECK); + checkSingleStatement(parse(sql = "check table a.a"), sql, StatementType.CHECK); + } + + @Test + public void testCreateStatement() { + String sql; + + checkSingleStatement(parse(sql = "create table a(a String) engine=Memory"), sql, StatementType.CREATE); + } + + @Test + public void testDeleteStatement() { + String sql; + + checkSingleStatement(parse(sql = "delete from a"), sql, StatementType.DELETE, "system", "a"); + checkSingleStatement(parse(sql = "delete from c.a where upper(a)=upper(lower(b))"), sql, StatementType.DELETE, + "c", "a"); + } + + @Test + public void testDescribeStatement() { + String sql; + + checkSingleStatement(parse(sql = "desc a"), sql, StatementType.DESCRIBE, "system", "columns"); + checkSingleStatement(parse(sql = "desc table a"), sql, StatementType.DESCRIBE, "system", "columns"); + checkSingleStatement(parse(sql = "describe table a.a"), sql, StatementType.DESCRIBE, "a", "columns"); + checkSingleStatement(parse(sql = "desc table table"), sql, StatementType.DESCRIBE, "system", "columns"); + // fix issue #614 + checkSingleStatement(parse(sql = "desc t1 t2"), sql, StatementType.DESCRIBE, "system", "columns"); + checkSingleStatement(parse(sql = "desc table t1 t2"), sql, StatementType.DESCRIBE, "system", "columns"); + checkSingleStatement(parse(sql = "desc table t1 as `t2`"), sql, StatementType.DESCRIBE, "system", "columns"); + } + + @Test + public void testDetachStatement() { + String sql; + + checkSingleStatement(parse(sql = "detach TABLE t"), sql, StatementType.DETACH); + checkSingleStatement(parse(sql = "detach TABLE if exists t.t on cluster 'cc'"), sql, StatementType.DETACH); + } + + @Test + public void testDropStatement() { + String sql; + + checkSingleStatement(parse(sql = "drop TEMPORARY table t"), sql, StatementType.DROP); + checkSingleStatement(parse(sql = "drop TABLE if exists t.t on cluster 'cc'"), sql, StatementType.DROP); + } + + @Test + public void testExistsStatement() { + String sql; + + checkSingleStatement(parse(sql = "EXISTS TEMPORARY TABLE a"), sql, StatementType.EXISTS); + checkSingleStatement(parse(sql = "EXISTS TABLE a.a"), sql, StatementType.EXISTS); + checkSingleStatement(parse(sql = "EXISTS DICTIONARY c"), sql, StatementType.EXISTS); + } + + @Test + public void testExplainStatement() { + String sql; + + checkSingleStatement(parse( + sql = "EXPLAIN SELECT sum(number) FROM numbers(10) UNION ALL SELECT sum(number) FROM numbers(10) ORDER BY sum(number) ASC FORMAT TSV"), + sql, StatementType.EXPLAIN); + checkSingleStatement(parse(sql = "EXPLAIN AST SELECT 1"), sql, StatementType.EXPLAIN); + checkSingleStatement(parse( + sql = "EXPLAIN SYNTAX SELECT * FROM system.numbers AS a, system.numbers AS b, system.numbers AS c"), + sql, StatementType.EXPLAIN); + } + + @Test + public void testGrantStatement() { + String sql; + + checkSingleStatement(parse(sql = "GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION"), sql, + StatementType.GRANT); + checkSingleStatement(parse(sql = "GRANT INSERT(x,y) ON db.table TO john"), sql, StatementType.GRANT); + } + + @Test + public void testInsertStatement() throws ParseException { + String sql; + + ClickHouseSqlStatement s = parse(sql = "insert into table test(a,b) Values (1,2)")[0]; + assertEquals(sql.substring(s.getStartPosition("values"), s.getEndPosition("VALUES")), "Values"); + assertEquals(sql.substring(0, s.getEndPosition("values")) + " (1,2)", sql); + + Pattern values = Pattern.compile("(?i)VALUES[\\s]*\\("); + int valuePosition = -1; + Matcher matcher = values.matcher(sql); + if (matcher.find()) { + valuePosition = matcher.start(); + } + assertEquals(s.getStartPosition("values"), valuePosition); + + checkSingleStatement(parse(sql = "insert into function null('a UInt8') values(1)"), sql, StatementType.INSERT); + checkSingleStatement(parse(sql = "insert into function null('a UInt8') values(1)(2)"), sql, + StatementType.INSERT); + checkSingleStatement(parse(sql = "insert into function null('a UInt8') select * from number(10)"), sql, + StatementType.INSERT); + checkSingleStatement(parse(sql = "insert into test2(a,b) values('values(',',')"), sql, StatementType.INSERT, + "system", "test2"); + checkSingleStatement(parse(sql = "INSERT INTO table t(a, b, c) values('1', ',', 'ccc')"), sql, + StatementType.INSERT, "system", "t"); + checkSingleStatement(parse(sql = "INSERT INTO table t(a, b, c) values('1', 2, 'ccc') (3,2,1)"), sql, + StatementType.INSERT, "system", "t"); + checkSingleStatement(parse(sql = "INSERT INTO table s.t select * from ttt"), sql, StatementType.INSERT, "s", + "t"); + checkSingleStatement(parse(sql = "INSERT INTO insert_select_testtable (* EXCEPT(b)) Values (2, 2)"), sql, + StatementType.INSERT, "system", "insert_select_testtable"); + + } + + @Test + public void testKillStatement() { + String sql; + + checkSingleStatement(parse(sql = "KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90'"), sql, + StatementType.KILL); + checkSingleStatement(parse( + sql = "KILL MUTATION WHERE database = 'default' AND table = 'table' AND mutation_id = 'mutation_3.txt' SYNC"), + sql, StatementType.KILL); + } + + @Test + public void testOptimizeStatement() { + String sql; + + checkSingleStatement(parse(sql = "OPTIMIZE TABLE a ON CLUSTER cluster PARTITION ID 'partition_id' FINAL"), sql, + StatementType.OPTIMIZE); + } + + @Test + public void testRenameStatement() { + String sql; + + checkSingleStatement(parse(sql = "RENAME TABLE table1 TO table2, table3 TO table4 ON CLUSTER cluster"), sql, + StatementType.RENAME); + checkSingleStatement(parse( + sql = "RENAME TABLE db1.table1 TO db2.table2, db2.table3 to db2.table4, db3.table5 to db2.table6 ON CLUSTER 'c'"), + sql, StatementType.RENAME); + } + + @Test + public void testRevokeStatement() { + String sql; + + checkSingleStatement(parse(sql = "REVOKE SELECT ON accounts.* FROM john"), sql, StatementType.REVOKE); + checkSingleStatement(parse(sql = "REVOKE SELECT(wage) ON accounts.staff FROM mira"), sql, StatementType.REVOKE); + } + + @Test + public void testSelectStatement() { + String sql; + + assertEquals(parse(sql = "select\n1"), new ClickHouseSqlStatement[] { + new ClickHouseSqlStatement(sql, StatementType.SELECT, null, null, "unknown", null, null, null, null) }); + assertEquals(parse(sql = "select\r\n1"), new ClickHouseSqlStatement[] { + new ClickHouseSqlStatement(sql, StatementType.SELECT, null, null, "unknown", null, null, null, null) }); + + assertEquals(parse(sql = "select 314 limit 5\nFORMAT JSONCompact;"), + new ClickHouseSqlStatement[] { new ClickHouseSqlStatement("select 314 limit 5\nFORMAT JSONCompact", + StatementType.SELECT, null, null, "unknown", "JSONCompact", null, null, null) }); + + checkSingleStatement(parse(sql = "select (())"), sql, StatementType.SELECT); + checkSingleStatement(parse(sql = "select []"), sql, StatementType.SELECT); + checkSingleStatement(parse(sql = "select [[]]"), sql, StatementType.SELECT); + checkSingleStatement(parse(sql = "select *"), sql, StatementType.SELECT); + checkSingleStatement(parse(sql = "select timezone()"), sql, StatementType.SELECT); + checkSingleStatement(parse(sql = "select @@version, $version"), sql, StatementType.SELECT); + checkSingleStatement(parse(sql = "select * from jdbc('db', 'schema', 'select 1')"), sql, StatementType.SELECT, + "system", "jdbc"); + checkSingleStatement(parse(sql = "select 1 as a1, a.a as a2, aa(a1, a2) a3, length(a3) as a4 from x"), sql, + StatementType.SELECT, "system", "x"); + checkSingleStatement(parse(sql = "select x.* from (select [1,2] a, (1,2,3) b, a[1], b.2) x"), sql, + StatementType.SELECT, "system", "x"); + checkSingleStatement(parse(sql = "select (3, [[1,2],[3,4]]) as a, (a.2)[2][1]"), sql, StatementType.SELECT); + checkSingleStatement( + parse(sql = "select 1,1.1,'\"''`a' a, \"'`\"\"a\" as b, (1 + `a`.a) c, null, inf i, nan as n"), sql, + StatementType.SELECT); + checkSingleStatement(parse(sql = "select 1 as select"), sql, StatementType.SELECT); + checkSingleStatement(parse(sql = "select 1, 2 a, 3 as b, 1+1-2*3/4, *, c.* from c a"), sql, + StatementType.SELECT, "system", "c"); + checkSingleStatement(parse(sql = "select 1 as select"), sql, StatementType.SELECT); + checkSingleStatement(parse( + sql = " -- cc\nselect 1 as `a.b`, a, 1+1, b from \"a\".`b` inner join a on a.abb/* \n\r\n1*/\n=2 and a.abb = c.a and a=1 and (k is null and j not in(1,2))"), + sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "SELECT idx, s FROM test.mymetadata WHERE idx = ?"), sql, StatementType.SELECT, + "test", "mymetadata"); + checkSingleStatement(parse(sql = "WITH 2 AS two SELECT two * two"), sql, StatementType.SELECT); + checkSingleStatement(parse( + sql = "SELECT i, array(toUnixTimestamp(dt_server[1])), array(toUnixTimestamp(dt_berlin[1])), array(toUnixTimestamp(dt_lax[1])) FROM test.fun_with_timezones_array"), + sql, StatementType.SELECT, "test", "fun_with_timezones_array"); + checkSingleStatement(parse(sql = "SELECT SUM(x) FROM t WHERE y = ? GROUP BY ?"), sql, StatementType.SELECT, + "system", "t"); + + assertEquals(parse(sql = loadSql("issue-441_with-totals.sql")), + new ClickHouseSqlStatement[] { new ClickHouseSqlStatement(sql, StatementType.SELECT, null, null, + "unknown", null, null, null, new HashMap() { + { + put("TOTALS", 208); + } + }) }); + assertEquals(parse(sql = loadSql("issue-555_custom-format.sql")), + new ClickHouseSqlStatement[] { new ClickHouseSqlStatement(sql, StatementType.SELECT, null, null, "wrd", + "CSVWithNames", null, null, null) }); + assertEquals(parse(sql = loadSql("with-clause.sql")), new ClickHouseSqlStatement[] { + new ClickHouseSqlStatement(sql, StatementType.SELECT, null, null, "unknown", null, null, null, null) }); + } + + @Test + public void testSetStatement() { + String sql; + + checkSingleStatement(parse(sql = "SET profile = 'my-profile', mutations_sync=1"), sql, StatementType.SET); + checkSingleStatement(parse(sql = "SET DEFAULT ROLE role1, role2, role3 TO user"), sql, StatementType.SET); + } + + @Test + public void testShowStatement() { + String sql; + + checkSingleStatement(parse(sql = "SHOW DATABASES LIKE '%de%'"), sql, StatementType.SHOW, "system", "databases"); + checkSingleStatement(parse(sql = "show tables from db"), sql, StatementType.SHOW, "system", "tables"); + checkSingleStatement(parse(sql = "show dictionaries from db"), sql, StatementType.SHOW, "system", + "dictionaries"); + } + + @Test + public void testSystemStatement() { + String sql; + + checkSingleStatement(parse(sql = "SYSTEM DROP REPLICA 'replica_name' FROM ZKPATH '/path/to/table/in/zk'"), sql, + StatementType.SYSTEM); + checkSingleStatement(parse(sql = "SYSTEM RESTART REPLICA db.replicated_merge_tree_family_table_name"), sql, + StatementType.SYSTEM); + } + + @Test + public void testTruncateStatement() { + String sql; + + checkSingleStatement(parse(sql = "truncate table a.b"), sql, StatementType.TRUNCATE, "a", "b"); + } + + @Test + public void testUpdateStatement() { + String sql; + + checkSingleStatement(parse(sql = "update a set a='1'"), sql, StatementType.UPDATE); + checkSingleStatement(parse(sql = "update a.a set `a`=2 where upper(a)=upper(lower(b))"), sql, + StatementType.UPDATE); + } + + @Test + public void testUseStatement() throws ParseException { + String sql; + checkSingleStatement(parse(sql = "use system"), sql, StatementType.USE); + } + + @Test + public void testWatchStatement() throws ParseException { + String sql; + checkSingleStatement(parse(sql = "watch system.processes"), sql, StatementType.WATCH); + } + + @Test + public void testComments() throws ParseException { + String sql; + checkSingleStatement(parse(sql = "select\n--something\n//else\n1/*2*/ from a.b"), sql, StatementType.SELECT, + "a", "b"); + + checkSingleStatement(parse(sql = "select 1/*/**/*/ from a.b"), sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select 1/*/1/**/*2*/ from a.b"), sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "SELECT /*/**/*/ 1 from a.b"), sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "SELECT /*a/*b*/c*/ 1 from a.b"), sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "SELECT /*ab/*cd*/ef*/ 1 from a.b"), sql, StatementType.SELECT, "a", "b"); + } + + @Test + public void testMultipleStatements() throws ParseException { + assertEquals(parse("use ab;;;select 1; ;\t;\r;\n"), new ClickHouseSqlStatement[] { + new ClickHouseSqlStatement("use ab", StatementType.USE, null, "ab", null, null, null, null, null), + new ClickHouseSqlStatement("select 1", StatementType.SELECT) }); + assertEquals(parse("select * from \"a;1\".`b;c`;;;select 1 as `a ; a`; ;\t;\r;\n"), + new ClickHouseSqlStatement[] { + new ClickHouseSqlStatement("select * from \"a;1\".`b;c`", StatementType.SELECT, null, "a;1", + "b;c", null, null, null, null), + new ClickHouseSqlStatement("select 1 as `a ; a`", StatementType.SELECT) }); + } + + @Test + public void testAlias() throws ParseException { + String sql; + checkSingleStatement(parse(sql = "select 1 as c, 2 b"), sql, StatementType.SELECT); + checkSingleStatement(parse(sql = "select 1 from a.b c"), sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select 1 select from a.b c"), sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select 1 from (select 2) b"), sql, StatementType.SELECT, "system", "b"); + checkSingleStatement(parse(sql = "select 1 from (select 2) as from"), sql, StatementType.SELECT, "system", + "from"); + checkSingleStatement(parse(sql = "select 1 from a.b c1, b.a c2"), sql, StatementType.SELECT, "a", "b"); + } + + @Test + public void testExpression() throws ParseException { + String sql; + checkSingleStatement(parse(sql = "SELECT a._ from a.b"), sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "SELECT 2 BETWEEN 1 + 1 AND 3 - 1 from a.b"), sql, StatementType.SELECT, "a", + "b"); + checkSingleStatement(parse(sql = "SELECT CASE WHEN 1 THEN 2 WHEN 3 THEN 4 ELSE 5 END from a.b"), sql, + StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select (1,2) a1, a1.1, a1 .1, a1 . 1 from a.b"), sql, StatementType.SELECT, + "a", "b"); + checkSingleStatement(parse(sql = "select -.0, +.0, -a from a.b"), sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select 1 and `a`.\"b\" c1, c1 or (c2 and c3), c4 ? c5 : c6 from a.b"), sql, + StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select [[[1,2],[3,4],[5,6]]] a, a[1][1][2] from a.b"), sql, + StatementType.SELECT, "a", "b"); + checkSingleStatement( + parse(sql = "select [[[[]]]], a[1][2][3], ([[1]] || [[2]])[2][1] ,func(1,2) [1] [2] [ 3 ] from a.b"), + sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select c.c1, c.c2 c, c.c3 as cc, c.c4.1.2 from a.b"), sql, + StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select - (select (1,).1) from a.b"), sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select 1.1e1,(1) . 1 , ((1,2)).1 .2 . 3 from a.b"), sql, StatementType.SELECT, + "a", "b"); + checkSingleStatement(parse(sql = "select a.b.c1, c1, b.c1 from a.b"), sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select date'2020-02-04', timestamp '2020-02-04' from a.b"), sql, + StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select count (), sum(c1), fake(a1, count(), (1+1)) from a.b"), sql, + StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select {}, {'a':'b', 'c':'1'} from a.b"), sql, StatementType.SELECT, "a", + "b"); + checkSingleStatement(parse(sql = "select [], [1,2], [ [1,2], [3,4] ] from a.b"), sql, StatementType.SELECT, "a", + "b"); + checkSingleStatement(parse(sql = "select 1+1-1*1/1 from a.b"), sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select (1+(1-1)*1/1)-1 from a.b"), sql, StatementType.SELECT, "a", "b"); + checkSingleStatement(parse(sql = "select (1+(1+(-1))*1/1)-(select (1,).1) from a.b"), sql, StatementType.SELECT, + "a", "b"); + } + + @Test + public void testFormat() throws ParseException { + String sql = "select 1 as format, format csv"; + ClickHouseSqlStatement[] stmts = parse(sql); + assertEquals(stmts.length, 1); + assertEquals(stmts[0].getSQL(), sql); + assertEquals(stmts[0].hasFormat(), false); + assertEquals(stmts[0].getFormat(), null); + + sql = "select 1 format csv"; + stmts = parse(sql); + assertEquals(stmts.length, 1); + assertEquals(stmts[0].getSQL(), sql); + assertEquals(stmts[0].hasFormat(), true); + assertEquals(stmts[0].getFormat(), "csv"); + + sql = "select 1 a, a.a b, a.a.a c, e.* except(e1), e.e.* except(e2), 'aaa' format, format csv from numbers(2) FORMAT CSVWithNames"; + stmts = parse(sql); + assertEquals(stmts.length, 1); + assertEquals(stmts[0].getSQL(), sql); + assertEquals(stmts[0].hasFormat(), true); + assertEquals(stmts[0].getFormat(), "CSVWithNames"); + } + + @Test + public void testOutfile() throws ParseException { + String sql = "select 1 into outfile '1.txt'"; + ClickHouseSqlStatement[] stmts = parse(sql); + assertEquals(stmts.length, 1); + assertEquals(stmts[0].getSQL(), sql); + assertEquals(stmts[0].hasOutfile(), true); + assertEquals(stmts[0].getOutfile(), "'1.txt'"); + + sql = "insert into outfile values(1,2,3)"; + stmts = parse(sql); + assertEquals(stmts.length, 1); + assertEquals(stmts[0].getSQL(), sql); + assertEquals(stmts[0].hasOutfile(), false); + assertEquals(stmts[0].getOutfile(), null); + } + + @Test + public void testWithTotals() throws ParseException { + String sql = "select 1 as with totals"; + ClickHouseSqlStatement[] stmts = parse(sql); + assertEquals(stmts.length, 1); + assertEquals(stmts[0].getSQL(), sql); + assertEquals(stmts[0].hasWithTotals(), false); + + sql = "select 1 with totals"; + stmts = parse(sql); + assertEquals(stmts.length, 1); + assertEquals(stmts[0].getSQL(), sql); + assertEquals(stmts[0].hasWithTotals(), true); + } + + @Test + public void testParameterHandling() throws ParseException { + String sql = "insert into table d.t(a1, a2, a3) values(?,?,?)"; + ClickHouseSqlStatement[] stmts = parse(sql); + assertEquals(stmts.length, 1); + assertEquals(stmts[0].getSQL(), sql); + + stmts = ClickHouseSqlParser.parse(sql, new ClickHouseProperties(), new ParseHandler() { + @Override + public String handleParameter(String cluster, String database, String table, int columnIndex) { + return String.valueOf(columnIndex); + } + }); + assertEquals(stmts.length, 1); + assertEquals(stmts[0].getSQL(), "insert into table d.t(a1, a2, a3) values(1,2,3)"); + } + + @Test + public void testMacroHandling() throws ParseException { + String sql = "select #listOfColumns #ignored from (#subQuery('1','2','3'))"; + ClickHouseSqlStatement[] stmts = parse(sql); + assertEquals(stmts.length, 1); + assertEquals(stmts[0].getSQL(), "select from ()"); + + stmts = ClickHouseSqlParser.parse(sql, new ClickHouseProperties(), new ParseHandler() { + @Override + public String handleMacro(String name, List parameters) { + if ("listOfColumns".equals(name)) { + return "a, b"; + } else if ("subQuery".equals(name)) { + return "select " + String.join("+", parameters); + } else { + return null; + } + } + }); + assertEquals(stmts.length, 1); + assertEquals(stmts[0].getSQL(), "select a, b from (select 1+2+3)"); + } + + @Test + public void testExtractDBAndTableName() { + String sql; + + checkSingleStatement(parse(sql = "SELECT 1 from table"), sql, StatementType.SELECT, "system", "table"); + checkSingleStatement(parse(sql = "SELECT 1 from table a"), sql, StatementType.SELECT, "system", "table"); + checkSingleStatement(parse(sql = "SELECT 1 from\ntable a"), sql, StatementType.SELECT, "system", "table"); + checkSingleStatement(parse(sql = "SELECT 1\nfrom\ntable a"), sql, StatementType.SELECT, "system", "table"); + checkSingleStatement(parse(sql = "SELECT 1\nFrom\ntable a"), sql, StatementType.SELECT, "system", "table"); + checkSingleStatement(parse(sql = "SELECT 1 from db.table a"), sql, StatementType.SELECT, "db", "table"); + checkSingleStatement(parse(sql = " SELECT 1 from \"db.table\" a"), sql, StatementType.SELECT, "system", + "db.table"); + checkSingleStatement(parse(sql = "SELECT 1 from `db.table` a"), sql, StatementType.SELECT, "system", + "db.table"); + checkSingleStatement(parse(sql = "from `db.table` a"), sql, StatementType.UNKNOWN, "system", "unknown"); + checkSingleStatement(parse(sql = " from `db.table` a"), sql, StatementType.UNKNOWN, "system", "unknown"); + checkSingleStatement(parse(sql = "ELECT from `db.table` a"), sql, StatementType.UNKNOWN, "system", "unknown"); + checkSingleStatement(parse(sql = "SHOW tables"), sql, StatementType.SHOW, "system", "tables"); + checkSingleStatement(parse(sql = "desc table1"), sql, StatementType.DESCRIBE, "system", "columns"); + checkSingleStatement(parse(sql = "DESC table1"), sql, StatementType.DESCRIBE, "system", "columns"); + checkSingleStatement(parse(sql = "SELECT 'from db.table a' from tab"), sql, StatementType.SELECT, "system", + "tab"); + checkSingleStatement(parse(sql = "SELECT"), sql, StatementType.UNKNOWN, "system", "unknown"); + checkSingleStatement(parse(sql = "S"), sql, StatementType.UNKNOWN, "system", "unknown"); + checkSingleStatement(parse(sql = ""), sql, StatementType.UNKNOWN, "system", "unknown"); + checkSingleStatement(parse(sql = " SELECT 1 from table from"), sql, StatementType.SELECT, "system", "table"); + checkSingleStatement(parse(sql = " SELECT 1 from table from"), sql, StatementType.SELECT, "system", "table"); + checkSingleStatement(parse(sql = "SELECT fromUnixTimestamp64Milli(time) as x from table"), sql, + StatementType.SELECT, "system", "table"); + checkSingleStatement(parse(sql = " SELECT fromUnixTimestamp64Milli(time)from table"), sql, StatementType.SELECT, + "system", "table"); + checkSingleStatement(parse(sql = "/*qq*/ SELECT fromUnixTimestamp64Milli(time)from table"), sql, + StatementType.SELECT, "system", "table"); + checkSingleStatement(parse(sql = " SELECTfromUnixTimestamp64Milli(time)from table"), sql, StatementType.UNKNOWN, + "system", "unknown"); + checkSingleStatement(parse(sql = " SELECT fromUnixTimestamp64Milli(time)from \".inner.a\""), sql, + StatementType.SELECT, "system", ".inner.a"); + checkSingleStatement(parse(sql = " SELECT fromUnixTimestamp64Milli(time)from db.`.inner.a`"), sql, + StatementType.SELECT, "db", ".inner.a"); + } + + // known issue + public void testTernaryOperator() { + String sql = "select x > 2 ? 'a' : 'b' from (select number as x from system.numbers limit ?)"; + ClickHouseSqlStatement[] stmts = parse(sql); + assertEquals(stmts.length, 1); + assertEquals(stmts[0].getStatementType(), StatementType.SELECT); + assertEquals(stmts[0].getParameters().size(), 1); + } + + static void parseAllSqlFiles(File f) throws IOException { + if (f.isDirectory()) { + File[] files = f.listFiles(); + for (File file : files) { + parseAllSqlFiles(file); + } + } else if (f.getName().endsWith(".sql")) { + StringBuilder sql = new StringBuilder(); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(new FileInputStream(f), StandardCharsets.UTF_8))) { + String line; + while ((line = br.readLine()) != null) { + sql.append(line).append("\n"); + } + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + + ClickHouseSqlParser p = new ClickHouseSqlParser(sql.toString(), null, null); + try { + p.sql(); + } catch (ParseException e) { + System.out.println(f.getAbsolutePath() + " -> " + e.getMessage()); + } catch (TokenMgrException e) { + System.out.println(f.getAbsolutePath() + " -> " + e.getMessage()); + } + } + } + + // TODO: add a sub-module points to ClickHouse/tests/queries? + public static void main(String[] args) throws Exception { + String chTestQueryDir = "D:/Sources/Github/ch/queries"; + if (args != null && args.length > 0) { + chTestQueryDir = args[0]; + } + chTestQueryDir = System.getProperty("chTestQueryDir", chTestQueryDir); + parseAllSqlFiles(new File(chTestQueryDir)); + } +} diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseCookieStoreProviderTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseCookieStoreProviderTest.java new file mode 100644 index 000000000..41500a9ae --- /dev/null +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseCookieStoreProviderTest.java @@ -0,0 +1,74 @@ +package ru.yandex.clickhouse.util; + +import org.testng.annotations.Test; +import ru.yandex.clickhouse.settings.ClickHouseProperties; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + +public class ClickHouseCookieStoreProviderTest { + ClickHouseCookieStoreProvider cookieStoreProvider = new ClickHouseCookieStoreProvider(); + + @Test + public void testCookieStoreProviderWithNullHost() { + ClickHouseProperties props = new ClickHouseProperties(); + props.setUseSharedCookieStore(true); + props.setPort(8080); + props.setDatabase("default"); + assertNull(cookieStoreProvider.getCookieStore(props)); + } + + @Test + public void testCookieStoreProviderWithInvalidPort() { + ClickHouseProperties props = new ClickHouseProperties(); + props.setUseSharedCookieStore(true); + props.setHost("127.0.0.1"); + props.setPort(0); + props.setDatabase("default"); + assertNull(cookieStoreProvider.getCookieStore(props)); + } + + @Test + public void testCookieStoreProviderWithNullDBName() { + ClickHouseProperties props = new ClickHouseProperties(); + props.setUseSharedCookieStore(true); + props.setHost("127.0.0.1"); + props.setPort(0); + assertNull(cookieStoreProvider.getCookieStore(props)); + } + + @Test + public void testCookieStoreProviderWithSameDBAndSharedCookieStore() { + ClickHouseProperties props = new ClickHouseProperties(); + props.setUseSharedCookieStore(true); + props.setHost("127.0.0.1"); + props.setPort(8080); + props.setDatabase("default"); + assertNotNull(cookieStoreProvider.getCookieStore(props)); + assertEquals(cookieStoreProvider.getCookieStore(props), cookieStoreProvider.getCookieStore(props)); + } + + @Test + public void testCookieStoreProviderWithPrivateCookieStore() { + ClickHouseProperties props = new ClickHouseProperties(); + props.setUseSharedCookieStore(false); + props.setHost("127.0.0.1"); + props.setPort(8080); + props.setDatabase("default"); + assertNull(cookieStoreProvider.getCookieStore(props)); + } + + @Test + public void testCookieStoreProviderWithDiffDB() { + ClickHouseProperties props1 = new ClickHouseProperties(); + props1.setUseSharedCookieStore(true); + props1.setHost("127.0.0.1"); + props1.setPort(8080); + props1.setDatabase("default1"); + ClickHouseProperties props2 = new ClickHouseProperties(props1); + props2.setDatabase("default2"); + assertNotEquals(cookieStoreProvider.getCookieStore(props1), cookieStoreProvider.getCookieStore(props2)); + } +} diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseHttpClientBuilderTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseHttpClientBuilderTest.java index 41502c0ee..8ecbcd1f8 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseHttpClientBuilderTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseHttpClientBuilderTest.java @@ -2,6 +2,7 @@ import org.apache.http.HttpHost; import org.apache.http.NoHttpResponseException; +import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.conn.HttpHostConnectException; import org.apache.http.impl.client.CloseableHttpClient; @@ -19,6 +20,9 @@ import ru.yandex.clickhouse.settings.ClickHouseProperties; +import java.time.Instant; + +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; @@ -101,6 +105,31 @@ public void testCreateClientContextOnlyPass() { "basic"); } + @Test + public void testHttpClientsWithSharedCookie() throws Exception { + ClickHouseProperties props = new ClickHouseProperties(); + props.setHost("localhost"); + props.setPort(mockServer.port()); + props.setDatabase("default"); + props.setUseSharedCookieStore(true); + CloseableHttpClient client = new ClickHouseHttpClientBuilder(props).buildClient(); + String cookie = "AWS-ALB=random-value-" + Instant.now().toEpochMilli(); + mockServer.stubFor(WireMock.get(WireMock.urlPathMatching("/cookie/get")) + .willReturn(WireMock.aResponse() + .withStatus(200) + .withHeader("set-cookie", cookie) + .withBody("OK"))); + HttpGet getCookie = new HttpGet(mockServer.baseUrl() + "/cookie/get"); + client.execute(getCookie); + CloseableHttpClient clientWithSharedCookieStore = new ClickHouseHttpClientBuilder(props).buildClient(); + props.setUseSharedCookieStore(false); + CloseableHttpClient clientWithPrivateCookieStore = new ClickHouseHttpClientBuilder(props).buildClient(); + HttpGet checkCookie = new HttpGet(mockServer.baseUrl() + "/cookie/check"); + clientWithPrivateCookieStore.execute(checkCookie); + mockServer.verify(getRequestedFor(WireMock.urlEqualTo("/cookie/check")).withoutHeader("cookie")); + clientWithSharedCookieStore.execute(checkCookie); + mockServer.verify(getRequestedFor(WireMock.urlEqualTo("/cookie/check")).withHeader("cookie", equalTo(cookie))); + } @Test(dataProvider = "authUserPassword") public void testHttpAuthParametersCombination(String authorization, String user, diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryInputStreamTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryInputStreamTest.java index 34ab234c2..1da6d70a5 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryInputStreamTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryInputStreamTest.java @@ -121,7 +121,7 @@ public void testOne() throws Exception { assertEquals(input.readUInt32(), 1492350000L); } - private ClickHouseRowBinaryInputStream prepareStream(byte[] input) { + private ClickHouseRowBinaryInputStream prepareStream(byte[] input) throws Exception { return new ClickHouseRowBinaryInputStream(new ByteArrayInputStream(input), TimeZone.getTimeZone("ETC"), new ClickHouseProperties()); } } diff --git a/clickhouse-jdbc/src/test/resources/log4j.properties b/clickhouse-jdbc/src/test/resources/log4j.properties index d274cc5cc..b6c544007 100644 --- a/clickhouse-jdbc/src/test/resources/log4j.properties +++ b/clickhouse-jdbc/src/test/resources/log4j.properties @@ -1,6 +1,6 @@ -log4j.rootLogger=DEBUG, STDOUT -log4j.category.ru.yandex.clickhouse=DEBUG -log4j.logger.org.apache.http=WARN +log4j.rootLogger=WARN, STDOUT +#log4j.category.ru.yandex.clickhouse=DEBUG +#log4j.logger.org.apache.http=DEBUG log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout log4j.appender.STDOUT.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.sss} [%t] [%-5p] {%c{1}:%L} - %m%n diff --git a/clickhouse-mysql-client/pom.xml b/clickhouse-mysql-client/pom.xml index 23430b915..87145a149 100644 --- a/clickhouse-mysql-client/pom.xml +++ b/clickhouse-mysql-client/pom.xml @@ -13,6 +13,10 @@ ${revision} jar + ${project.artifactId} + MySQL client for ClickHouse + https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-mysql-client + 2.7.2 8.0.23 diff --git a/clickhouse-native-client/pom.xml b/clickhouse-native-client/pom.xml index 996318396..5969700e6 100644 --- a/clickhouse-native-client/pom.xml +++ b/clickhouse-native-client/pom.xml @@ -13,6 +13,10 @@ ${revision} jar + ${project.artifactId} + Native client for ClickHouse + https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-native-client + ${parent.groupId} diff --git a/pom.xml b/pom.xml index efe899718..194135348 100644 --- a/pom.xml +++ b/pom.xml @@ -74,7 +74,7 @@ - 0.3.0-SNAPSHOT + 0.3.1-SNAPSHOT 2021 UTF-8 UTF-8 @@ -85,7 +85,7 @@ 1.7.1 2.9.10 2.9.10.8 - 0.9.9 + 0.9.10 1.7.30 1.10.19 2.27.2 @@ -93,6 +93,7 @@ 6.14.3 3.3.0 3.0.0-M1 + 1.2.7 1.6.8 1.6 3.8.1 @@ -109,10 +110,10 @@ ${skipTests} clickhouse_clickhouse-jdbc - ${artifactId} + ${project.artifactId} clickhouse https://sonarcloud.io - 0.3.0 + 0.3.1 @@ -148,7 +149,7 @@ ${lz4.version} - com.github.RoaringBitmap + org.roaringbitmap RoaringBitmap ${roaring-bitmap.version} @@ -274,8 +275,39 @@ false + + org.codehaus.mojo + flatten-maven-plugin + ${flatten-plugin.version} + + true + ossrh + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + + org.codehaus.mojo + flatten-maven-plugin + + @@ -401,12 +433,4 @@ - - - - - jitpack.io - https://jitpack.io - -