diff --git a/docs/src/main/sphinx/connector/clickhouse.rst b/docs/src/main/sphinx/connector/clickhouse.rst index df0c5d247824..89d879383665 100644 --- a/docs/src/main/sphinx/connector/clickhouse.rst +++ b/docs/src/main/sphinx/connector/clickhouse.rst @@ -160,26 +160,30 @@ Type mapping The data type mappings are as follows: -================= =============== =================================================================================================== -ClickHouse Trino Notes -================= =============== =================================================================================================== -``Int8`` ``TINYINT`` ``TINYINT``, ``BOOL``, ``BOOLEAN`` and ``INT1`` are aliases of ``Int8`` -``Int16`` ``SMALLINT`` ``SMALLINT`` and ``INT2`` are aliases of ``Int16`` -``Int32`` ``INTEGER`` ``INT``, ``INT4`` and ``INTEGER`` are aliases of ``Int32`` -``Int64`` ``BIGINT`` ``BIGINT`` is an alias of ``Int64`` -``Float32`` ``REAL`` ``FLOAT`` is an alias of ``Float32`` -``Float64`` ``DOUBLE`` ``DOUBLE`` is an alias of ``Float64`` +================= ================= =================================================================================================== +ClickHouse Trino Notes +================= ================= =================================================================================================== +``Int8`` ``TINYINT`` ``TINYINT``, ``BOOL``, ``BOOLEAN`` and ``INT1`` are aliases of ``Int8`` +``Int16`` ``SMALLINT`` ``SMALLINT`` and ``INT2`` are aliases of ``Int16`` +``Int32`` ``INTEGER`` ``INT``, ``INT4`` and ``INTEGER`` are aliases of ``Int32`` +``Int64`` ``BIGINT`` ``BIGINT`` is an alias of ``Int64`` +``UInt8`` ``SMALLINT`` +``UInt16`` ``INTEGER`` +``UInt32`` ``BIGINT`` +``UInt64`` ``DECIMAL(20,0)`` +``Float32`` ``REAL`` ``FLOAT`` is an alias of ``Float32`` +``Float64`` ``DOUBLE`` ``DOUBLE`` is an alias of ``Float64`` ``Decimal`` ``DECIMAL`` -``FixedString`` ``VARBINARY`` Enabling ``clickhouse.map-string-as-varchar`` config property changes the mapping to ``VARCHAR`` -``String`` ``VARBINARY`` Enabling ``clickhouse.map-string-as-varchar`` config property changes the mapping to ``VARCHAR`` +``FixedString`` ``VARBINARY`` Enabling ``clickhouse.map-string-as-varchar`` config property changes the mapping to ``VARCHAR`` +``String`` ``VARBINARY`` Enabling ``clickhouse.map-string-as-varchar`` config property changes the mapping to ``VARCHAR`` ``Date`` ``DATE`` ``DateTime`` ``TIMESTAMP`` ``IPv4`` ``IPADDRESS`` ``IPv6`` ``IPADDRESS`` ``Enum8`` ``VARCHAR`` ``Enum16`` ``VARCHAR`` -``UUID`` ``UUID`` -================= =============== =================================================================================================== +``UUID`` ``UUID`` +================= ================= =================================================================================================== .. include:: jdbc-type-mapping.fragment diff --git a/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClient.java b/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClient.java index bcc6e6289762..70c7bcda59c2 100644 --- a/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClient.java +++ b/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClient.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; +import com.google.common.primitives.Shorts; import io.airlift.slice.Slice; import io.trino.plugin.base.aggregation.AggregateFunctionRewriter; import io.trino.plugin.base.aggregation.AggregateFunctionRule; @@ -33,6 +34,7 @@ import io.trino.plugin.jdbc.JdbcTableHandle; import io.trino.plugin.jdbc.JdbcTypeHandle; import io.trino.plugin.jdbc.LongWriteFunction; +import io.trino.plugin.jdbc.ObjectWriteFunction; import io.trino.plugin.jdbc.QueryBuilder; import io.trino.plugin.jdbc.RemoteTableName; import io.trino.plugin.jdbc.SliceWriteFunction; @@ -53,6 +55,7 @@ import io.trino.spi.type.CharType; import io.trino.spi.type.DecimalType; import io.trino.spi.type.Decimals; +import io.trino.spi.type.Int128; import io.trino.spi.type.StandardTypes; import io.trino.spi.type.Type; import io.trino.spi.type.TypeManager; @@ -64,6 +67,9 @@ import javax.inject.Inject; import java.io.UncheckedIOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.MathContext; import java.net.InetAddress; import java.net.UnknownHostException; import java.sql.Connection; @@ -108,6 +114,7 @@ import static io.trino.plugin.jdbc.StandardColumnMappings.doubleWriteFunction; import static io.trino.plugin.jdbc.StandardColumnMappings.integerColumnMapping; import static io.trino.plugin.jdbc.StandardColumnMappings.integerWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.longDecimalReadFunction; import static io.trino.plugin.jdbc.StandardColumnMappings.longDecimalWriteFunction; import static io.trino.plugin.jdbc.StandardColumnMappings.realWriteFunction; import static io.trino.plugin.jdbc.StandardColumnMappings.shortDecimalWriteFunction; @@ -145,9 +152,11 @@ import static java.lang.Math.floorDiv; import static java.lang.Math.floorMod; import static java.lang.Math.max; +import static java.lang.Math.toIntExact; import static java.lang.String.format; import static java.lang.String.join; import static java.lang.System.arraycopy; +import static java.math.RoundingMode.UNNECESSARY; import static java.time.ZoneOffset.UTC; import static java.util.Locale.ENGLISH; @@ -156,6 +165,19 @@ public class ClickHouseClient { private static final Splitter TABLE_PROPERTY_SPLITTER = Splitter.on(',').omitEmptyStrings().trimResults(); + private static final long UINT8_MIN_VALUE = 0L; + private static final long UINT8_MAX_VALUE = 255L; + + private static final long UINT16_MIN_VALUE = 0L; + private static final long UINT16_MAX_VALUE = 65535L; + + private static final long UINT32_MIN_VALUE = 0L; + private static final long UINT32_MAX_VALUE = 4294967295L; + + private static final DecimalType UINT64_TYPE = createDecimalType(20, 0); + private static final BigDecimal UINT64_MIN_VALUE = BigDecimal.ZERO; + private static final BigDecimal UINT64_MAX_VALUE = new BigDecimal("18446744073709551615"); + private static final long MIN_SUPPORTED_DATE_EPOCH = LocalDate.parse("1970-01-01").toEpochDay(); private static final long MAX_SUPPORTED_DATE_EPOCH = LocalDate.parse("2106-02-07").toEpochDay(); // The max date is '2148-12-31' in new ClickHouse version @@ -463,6 +485,17 @@ public Optional toColumnMapping(ConnectorSession session, Connect ClickHouseColumn column = ClickHouseColumn.of("", jdbcTypeName); ClickHouseDataType columnDataType = column.getDataType(); switch (columnDataType) { + case UInt8: + return Optional.of(ColumnMapping.longMapping(SMALLINT, ResultSet::getShort, uInt8WriteFunction())); + case UInt16: + return Optional.of(ColumnMapping.longMapping(INTEGER, ResultSet::getInt, uInt16WriteFunction())); + case UInt32: + return Optional.of(ColumnMapping.longMapping(BIGINT, ResultSet::getLong, uInt32WriteFunction())); + case UInt64: + return Optional.of(ColumnMapping.objectMapping( + UINT64_TYPE, + longDecimalReadFunction(UINT64_TYPE, UNNECESSARY), + uInt64WriteFunction())); case IPv4: return Optional.of(ipAddressColumnMapping("IPv4StringToNum(?)")); case IPv6: @@ -627,6 +660,54 @@ private Optional formatProperty(List prop) return Optional.of("(" + String.join(",", prop) + ")"); } + private static LongWriteFunction uInt8WriteFunction() + { + return (statement, index, value) -> { + // ClickHouse stores incorrect results when the values are out of supported range. + if (value < UINT8_MIN_VALUE || value > UINT8_MAX_VALUE) { + throw new TrinoException(INVALID_ARGUMENTS, format("Value must be between %s and %s in ClickHouse: %s", UINT8_MIN_VALUE, UINT8_MAX_VALUE, value)); + } + statement.setShort(index, Shorts.checkedCast(value)); + }; + } + + private static LongWriteFunction uInt16WriteFunction() + { + return (statement, index, value) -> { + // ClickHouse stores incorrect results when the values are out of supported range. + if (value < UINT16_MIN_VALUE || value > UINT16_MAX_VALUE) { + throw new TrinoException(INVALID_ARGUMENTS, format("Value must be between %s and %s in ClickHouse: %s", UINT16_MIN_VALUE, UINT16_MAX_VALUE, value)); + } + statement.setInt(index, toIntExact(value)); + }; + } + + private static LongWriteFunction uInt32WriteFunction() + { + return (preparedStatement, parameterIndex, value) -> { + // ClickHouse stores incorrect results when the values are out of supported range. + if (value < UINT32_MIN_VALUE || value > UINT32_MAX_VALUE) { + throw new TrinoException(INVALID_ARGUMENTS, format("Value must be between %s and %s in ClickHouse: %s", UINT32_MIN_VALUE, UINT32_MAX_VALUE, value)); + } + preparedStatement.setLong(parameterIndex, value); + }; + } + + private static ObjectWriteFunction uInt64WriteFunction() + { + return ObjectWriteFunction.of( + Int128.class, + (statement, index, value) -> { + BigInteger unscaledValue = value.toBigInteger(); + BigDecimal bigDecimal = new BigDecimal(unscaledValue, UINT64_TYPE.getScale(), new MathContext(UINT64_TYPE.getPrecision())); + // ClickHouse stores incorrect results when the values are out of supported range. + if (bigDecimal.compareTo(UINT64_MIN_VALUE) < 0 || bigDecimal.compareTo(UINT64_MAX_VALUE) > 0) { + throw new TrinoException(INVALID_ARGUMENTS, format("Value must be between %s and %s in ClickHouse: %s", UINT64_MIN_VALUE, UINT64_MAX_VALUE, bigDecimal)); + } + statement.setBigDecimal(index, bigDecimal); + }); + } + private static ColumnMapping dateColumnMappingUsingLocalDate() { return ColumnMapping.longMapping( diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseTypeMapping.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseTypeMapping.java index 87793bbd2f84..8b893dbddaf1 100644 --- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseTypeMapping.java +++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseTypeMapping.java @@ -230,6 +230,150 @@ public void testUnsupportedBigint() .execute(getQueryRunner(), clickhouseCreateAndInsert("tpch.test_unsupported_bigint")); } + @Test + public void testUint8() + { + SqlDataTypeTest.create() + .addRoundTrip("UInt8", "0", SMALLINT, "SMALLINT '0'") // min value in ClickHouse + .addRoundTrip("UInt8", "255", SMALLINT, "SMALLINT '255'") // max value in ClickHouse + .addRoundTrip("Nullable(UInt8)", "NULL", SMALLINT, "CAST(null AS SMALLINT)") + .execute(getQueryRunner(), clickhouseCreateAndInsert("tpch.test_uint8")); + + SqlDataTypeTest.create() + .addRoundTrip("UInt8", "0", SMALLINT, "SMALLINT '0'") // min value in ClickHouse + .addRoundTrip("UInt8", "255", SMALLINT, "SMALLINT '255'") // max value in ClickHouse + .addRoundTrip("Nullable(UInt8)", "NULL", SMALLINT, "CAST(null AS SMALLINT)") + .execute(getQueryRunner(), clickhouseCreateTrinoInsert("tpch.test_uint8")); + } + + @Test + public void testUnsupportedUint8() + { + // ClickHouse stores incorrect results when the values are out of supported range. This test should be fixed when ClickHouse changes the behavior. + SqlDataTypeTest.create() + .addRoundTrip("UInt8", "-1", SMALLINT, "SMALLINT '255'") + .addRoundTrip("UInt8", "256", SMALLINT, "SMALLINT '0'") + .execute(getQueryRunner(), clickhouseCreateAndInsert("tpch.test_unsupported_uint8")); + + // Prevent writing incorrect results in the connector + try (TestTable table = new TestTable(clickhouseServer::execute, "tpch.test_unsupported_uint8", "(value UInt8) ENGINE=Log")) { + assertQueryFails( + format("INSERT INTO %s VALUES (-1)", table.getName()), + "Value must be between 0 and 255 in ClickHouse: -1"); + assertQueryFails( + format("INSERT INTO %s VALUES (256)", table.getName()), + "Value must be between 0 and 255 in ClickHouse: 256"); + } + } + + @Test + public void testUint16() + { + SqlDataTypeTest.create() + .addRoundTrip("UInt16", "0", INTEGER, "0") // min value in ClickHouse + .addRoundTrip("UInt16", "65535", INTEGER, "65535") // max value in ClickHouse + .addRoundTrip("Nullable(UInt16)", "NULL", INTEGER, "CAST(null AS INTEGER)") + .execute(getQueryRunner(), clickhouseCreateAndInsert("tpch.test_uint16")); + + SqlDataTypeTest.create() + .addRoundTrip("UInt16", "0", INTEGER, "0") // min value in ClickHouse + .addRoundTrip("UInt16", "65535", INTEGER, "65535") // max value in ClickHouse + .addRoundTrip("Nullable(UInt16)", "NULL", INTEGER, "CAST(null AS INTEGER)") + .execute(getQueryRunner(), clickhouseCreateTrinoInsert("tpch.test_uint16")); + } + + @Test + public void testUnsupportedUint16() + { + // ClickHouse stores incorrect results when the values are out of supported range. This test should be fixed when ClickHouse changes the behavior. + SqlDataTypeTest.create() + .addRoundTrip("UInt16", "-1", INTEGER, "65535") + .addRoundTrip("UInt16", "65536", INTEGER, "0") + .execute(getQueryRunner(), clickhouseCreateAndInsert("tpch.test_unsupported_uint16")); + + // Prevent writing incorrect results in the connector + try (TestTable table = new TestTable(clickhouseServer::execute, "tpch.test_unsupported_uint16", "(value UInt16) ENGINE=Log")) { + assertQueryFails( + format("INSERT INTO %s VALUES (-1)", table.getName()), + "Value must be between 0 and 65535 in ClickHouse: -1"); + assertQueryFails( + format("INSERT INTO %s VALUES (65536)", table.getName()), + "Value must be between 0 and 65535 in ClickHouse: 65536"); + } + } + + @Test + public void testUint32() + { + SqlDataTypeTest.create() + .addRoundTrip("UInt32", "0", BIGINT, "BIGINT '0'") // min value in ClickHouse + .addRoundTrip("UInt32", "4294967295", BIGINT, "BIGINT '4294967295'") // max value in ClickHouse + .addRoundTrip("Nullable(UInt32)", "NULL", BIGINT, "CAST(null AS BIGINT)") + .execute(getQueryRunner(), clickhouseCreateAndInsert("tpch.test_uint32")); + + SqlDataTypeTest.create() + .addRoundTrip("UInt32", "BIGINT '0'", BIGINT, "BIGINT '0'") // min value in ClickHouse + .addRoundTrip("UInt32", "BIGINT '4294967295'", BIGINT, "BIGINT '4294967295'") // max value in ClickHouse + .addRoundTrip("Nullable(UInt32)", "NULL", BIGINT, "CAST(null AS BIGINT)") + .execute(getQueryRunner(), clickhouseCreateTrinoInsert("tpch.test_uint32")); + } + + @Test + public void testUnsupportedUint32() + { + // ClickHouse stores incorrect results when the values are out of supported range. This test should be fixed when ClickHouse changes the behavior. + SqlDataTypeTest.create() + .addRoundTrip("UInt32", "-1", BIGINT, "BIGINT '4294967295'") + .addRoundTrip("UInt32", "4294967296", BIGINT, "BIGINT '0'") + .execute(getQueryRunner(), clickhouseCreateAndInsert("tpch.test_unsupported_uint32")); + + // Prevent writing incorrect results in the connector + try (TestTable table = new TestTable(clickhouseServer::execute, "tpch.test_unsupported_uint32", "(value UInt32) ENGINE=Log")) { + assertQueryFails( + format("INSERT INTO %s VALUES (CAST('-1' AS BIGINT))", table.getName()), + "Value must be between 0 and 4294967295 in ClickHouse: -1"); + assertQueryFails( + format("INSERT INTO %s VALUES (CAST('4294967296' AS BIGINT))", table.getName()), + "Value must be between 0 and 4294967295 in ClickHouse: 4294967296"); + } + } + + @Test + public void testUint64() + { + SqlDataTypeTest.create() + .addRoundTrip("UInt64", "0", createDecimalType(20), "CAST('0' AS decimal(20, 0))") // min value in ClickHouse + .addRoundTrip("UInt64", "18446744073709551615", createDecimalType(20), "CAST('18446744073709551615' AS decimal(20, 0))") // max value in ClickHouse + .addRoundTrip("Nullable(UInt64)", "NULL", createDecimalType(20), "CAST(null AS decimal(20, 0))") + .execute(getQueryRunner(), clickhouseCreateAndInsert("tpch.test_uint64")); + + SqlDataTypeTest.create() + .addRoundTrip("UInt64", "CAST('0' AS decimal(20, 0))", createDecimalType(20), "CAST('0' AS decimal(20, 0))") // min value in ClickHouse + .addRoundTrip("UInt64", "CAST('18446744073709551615' AS decimal(20, 0))", createDecimalType(20), "CAST('18446744073709551615' AS decimal(20, 0))") // max value in ClickHouse + .addRoundTrip("Nullable(UInt64)", "NULL", createDecimalType(20), "CAST(null AS decimal(20, 0))") + .execute(getQueryRunner(), clickhouseCreateTrinoInsert("tpch.test_uint64")); + } + + @Test + public void testUnsupportedUint64() + { + // ClickHouse stores incorrect results when the values are out of supported range. This test should be fixed when ClickHouse changes the behavior. + SqlDataTypeTest.create() + .addRoundTrip("UInt64", "-1", createDecimalType(20), "CAST('18446744073709551615' AS decimal(20, 0))") + .addRoundTrip("UInt64", "18446744073709551616", createDecimalType(20), "CAST('0' AS decimal(20, 0))") + .execute(getQueryRunner(), clickhouseCreateAndInsert("tpch.test_unsupported_uint64")); + + // Prevent writing incorrect results in the connector + try (TestTable table = new TestTable(clickhouseServer::execute, "tpch.test_unsupported_uint64", "(value UInt64) ENGINE=Log")) { + assertQueryFails( + format("INSERT INTO %s VALUES (CAST('-1' AS decimal(20, 0)))", table.getName()), + "Value must be between 0 and 18446744073709551615 in ClickHouse: -1"); + assertQueryFails( + format("INSERT INTO %s VALUES (CAST('18446744073709551616' AS decimal(20, 0)))", table.getName()), + "Value must be between 0 and 18446744073709551615 in ClickHouse: 18446744073709551616"); + } + } + @Test public void testReal() {