diff --git a/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClient.java b/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClient.java index 7ab40471d762..63f1817a5477 100644 --- a/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClient.java +++ b/plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClient.java @@ -67,6 +67,8 @@ import java.sql.SQLException; import java.sql.Types; import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Map; import java.util.Optional; @@ -100,8 +102,8 @@ import static io.trino.plugin.jdbc.StandardColumnMappings.shortDecimalWriteFunction; import static io.trino.plugin.jdbc.StandardColumnMappings.smallintColumnMapping; import static io.trino.plugin.jdbc.StandardColumnMappings.smallintWriteFunction; -import static io.trino.plugin.jdbc.StandardColumnMappings.timestampColumnMapping; import static io.trino.plugin.jdbc.StandardColumnMappings.timestampColumnMappingUsingSqlTimestampWithRounding; +import static io.trino.plugin.jdbc.StandardColumnMappings.timestampReadFunction; import static io.trino.plugin.jdbc.StandardColumnMappings.tinyintColumnMapping; import static io.trino.plugin.jdbc.StandardColumnMappings.tinyintWriteFunction; import static io.trino.plugin.jdbc.StandardColumnMappings.varbinaryColumnMapping; @@ -122,15 +124,20 @@ import static io.trino.spi.type.SmallintType.SMALLINT; import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; import static io.trino.spi.type.TimestampType.TIMESTAMP_SECONDS; +import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_SECOND; +import static io.trino.spi.type.Timestamps.NANOSECONDS_PER_MICROSECOND; import static io.trino.spi.type.TinyintType.TINYINT; import static io.trino.spi.type.UuidType.javaUuidToTrinoUuid; import static io.trino.spi.type.UuidType.trinoUuidToJavaUuid; import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; import static java.lang.Float.floatToRawIntBits; +import static java.lang.Math.floorDiv; +import static java.lang.Math.floorMod; import static java.lang.Math.max; import static java.lang.String.format; import static java.lang.String.join; import static java.lang.System.arraycopy; +import static java.time.ZoneOffset.UTC; public class ClickHouseClient extends BaseJdbcClient @@ -139,6 +146,11 @@ public class ClickHouseClient private static final long MIN_SUPPORTED_DATE_EPOCH = LocalDate.parse("1970-01-01").toEpochDay(); private static final long MAX_SUPPORTED_DATE_EPOCH = LocalDate.parse("2106-02-07").toEpochDay(); // The max date is '2148-12-31' in new ClickHouse version + private static final LocalDateTime MIN_SUPPORTED_TIMESTAMP = LocalDateTime.parse("1970-01-01T00:00:00"); + private static final LocalDateTime MAX_SUPPORTED_TIMESTAMP = LocalDateTime.parse("2105-12-31T23:59:59"); + private static final long MIN_SUPPORTED_TIMESTAMP_EPOCH = MIN_SUPPORTED_TIMESTAMP.toEpochSecond(UTC); + private static final long MAX_SUPPORTED_TIMESTAMP_EPOCH = MAX_SUPPORTED_TIMESTAMP.toEpochSecond(UTC); + private final AggregateFunctionRewriter aggregateFunctionRewriter; private final Type uuidType; private final Type ipAddressType; @@ -468,7 +480,10 @@ public Optional toColumnMapping(ConnectorSession session, Connect case Types.TIMESTAMP: if (jdbcTypeName.equals("DateTime")) { verify(typeHandle.getRequiredDecimalDigits() == 0, "Expected 0 as timestamp precision, but got %s", typeHandle.getRequiredDecimalDigits()); - return Optional.of(timestampColumnMapping(TIMESTAMP_SECONDS)); + return Optional.of(ColumnMapping.longMapping( + TIMESTAMP_SECONDS, + timestampReadFunction(TIMESTAMP_SECONDS), + timestampSecondsWriteFunction())); } // TODO (https://github.com/trinodb/trino/issues/10537) Add support for Datetime64 type return Optional.of(timestampColumnMappingUsingSqlTimestampWithRounding(TIMESTAMP_MILLIS)); @@ -522,6 +537,9 @@ public WriteMapping toWriteMapping(ConnectorSession session, Type type) // TODO (https://github.com/trinodb/trino/issues/10055) Deny unsupported dates to prevent inserting wrong values. 2106-02-07 is max value in version 20.8 return WriteMapping.longMapping("Date", dateWriteFunctionUsingLocalDate()); } + if (type == TIMESTAMP_SECONDS) { + return WriteMapping.longMapping("DateTime", timestampSecondsWriteFunction()); + } if (type.equals(uuidType)) { return WriteMapping.sliceMapping("UUID", uuidWriteFunction()); } @@ -569,7 +587,26 @@ private static void verifySupportedDate(long value) { // Deny unsupported dates eagerly to prevent unexpected results. ClickHouse stores '1970-01-01' when the date is out of supported range. if (value < MIN_SUPPORTED_DATE_EPOCH || value > MAX_SUPPORTED_DATE_EPOCH) { - throw new TrinoException(INVALID_ARGUMENTS, format("Date must be between %s and %s: %s", LocalDate.ofEpochDay(MIN_SUPPORTED_DATE_EPOCH), LocalDate.ofEpochDay(MAX_SUPPORTED_DATE_EPOCH), LocalDate.ofEpochDay(value))); + throw new TrinoException(INVALID_ARGUMENTS, format("Date must be between %s and %s in ClickHouse: %s", LocalDate.ofEpochDay(MIN_SUPPORTED_DATE_EPOCH), LocalDate.ofEpochDay(MAX_SUPPORTED_DATE_EPOCH), LocalDate.ofEpochDay(value))); + } + } + + private static LongWriteFunction timestampSecondsWriteFunction() + { + return (statement, index, value) -> { + long epochSecond = floorDiv(value, MICROSECONDS_PER_SECOND); + int nanoFraction = floorMod(value, MICROSECONDS_PER_SECOND) * NANOSECONDS_PER_MICROSECOND; + verify(nanoFraction == 0, "Nanos of second must be zero: '%s'", value); + verifySupportedTimestamp(epochSecond); + statement.setObject(index, LocalDateTime.ofEpochSecond(epochSecond, 0, UTC)); + }; + } + + private static void verifySupportedTimestamp(long epochSecond) + { + if (epochSecond < MIN_SUPPORTED_TIMESTAMP_EPOCH || epochSecond > MAX_SUPPORTED_TIMESTAMP_EPOCH) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss"); + throw new TrinoException(INVALID_ARGUMENTS, format("Timestamp must be between %s and %s in ClickHouse: %s", MIN_SUPPORTED_TIMESTAMP.format(formatter), MAX_SUPPORTED_TIMESTAMP.format(formatter), LocalDateTime.ofEpochSecond(epochSecond, 0, UTC).format(formatter))); } } diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseTypeMapping.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseTypeMapping.java index 0e3e623e9047..f96b84e1504a 100644 --- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseTypeMapping.java +++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseTypeMapping.java @@ -421,10 +421,10 @@ public void testUnsupportedDate() try (TestTable table = new TestTable(getQueryRunner()::execute, "test_unsupported_date", "(dt date)")) { assertQueryFails( format("INSERT INTO %s VALUES (DATE '1969-12-31')", table.getName()), - "Date must be between 1970-01-01 and 2106-02-07: 1969-12-31"); + "Date must be between 1970-01-01 and 2106-02-07 in ClickHouse: 1969-12-31"); assertQueryFails( format("INSERT INTO %s VALUES (DATE '2106-02-08')", table.getName()), - "Date must be between 1970-01-01 and 2106-02-07: 2106-02-08"); + "Date must be between 1970-01-01 and 2106-02-07 in ClickHouse: 2106-02-08"); } } @@ -435,7 +435,16 @@ public void testTimestamp(ZoneId sessionZone) .setTimeZoneKey(TimeZoneKey.getTimeZoneKey(sessionZone.getId())) .build(); - // TODO (https://github.com/trinodb/trino/issues/10538) Support writing timestamp and datetime types in ClickHouse connector + SqlDataTypeTest.create() + .addRoundTrip("timestamp(0)", "timestamp '1970-01-01 00:00:00'", createTimestampType(0), "TIMESTAMP '1970-01-01 00:00:00'") // min value in ClickHouse + .addRoundTrip("timestamp(0)", "timestamp '1986-01-01 00:13:07'", createTimestampType(0), "TIMESTAMP '1986-01-01 00:13:07'") // time gap in Kathmandu + .addRoundTrip("timestamp(0)", "timestamp '2018-03-25 03:17:17'", createTimestampType(0), "TIMESTAMP '2018-03-25 03:17:17'") // time gap in Vilnius + .addRoundTrip("timestamp(0)", "timestamp '2018-10-28 01:33:17'", createTimestampType(0), "TIMESTAMP '2018-10-28 01:33:17'") // time doubled in JVM zone + .addRoundTrip("timestamp(0)", "timestamp '2018-10-28 03:33:33'", createTimestampType(0), "TIMESTAMP '2018-10-28 03:33:33'") // time double in Vilnius + .addRoundTrip("timestamp(0)", "timestamp '2105-12-31 23:59:59'", createTimestampType(0), "TIMESTAMP '2105-12-31 23:59:59'") // max value in ClickHouse + .execute(getQueryRunner(), session, trinoCreateAsSelect("tpch.test_timestamp")) + .execute(getQueryRunner(), session, trinoCreateAndInsert("tpch.test_timestamp")); + addTimestampRoundTrips("timestamp") .execute(getQueryRunner(), session, clickhouseCreateAndInsert("tpch.test_timestamp")); addTimestampRoundTrips("datetime") @@ -452,10 +461,28 @@ private SqlDataTypeTest addTimestampRoundTrips(String inputType) .addRoundTrip(inputType, "'2018-10-28 01:33:17'", createTimestampType(0), "TIMESTAMP '2018-10-28 01:33:17'") // time doubled in JVM zone .addRoundTrip(inputType, "'2018-10-28 03:33:33'", createTimestampType(0), "TIMESTAMP '2018-10-28 03:33:33'") // time double in Vilnius .addRoundTrip(inputType, "'2105-12-31 23:59:59'", createTimestampType(0), "TIMESTAMP '2105-12-31 23:59:59'") // max value in ClickHouse - .addRoundTrip(inputType, "'2106-01-01 00:00:00'", createTimestampType(0), "TIMESTAMP '2106-01-01 00:00:00'") // unsupported timestamp become 1970-01-01 23:59:59 .addRoundTrip(format("Nullable(%s)", inputType), "NULL", createTimestampType(0), "CAST(NULL AS TIMESTAMP(0))"); } + @Test + public void testUnsupportedTimestamp() + { + try (TestTable table = new TestTable(getQueryRunner()::execute, "test_unsupported_timestamp", "(dt timestamp(0))")) { + assertQueryFails( + format("INSERT INTO %s VALUES (TIMESTAMP '-9999-12-31 23:59:59')", table.getName()), + "Timestamp must be between 1970-01-01 00:00:00 and 2105-12-31 23:59:59 in ClickHouse: -9999-12-31 23:59:59"); + assertQueryFails( + format("INSERT INTO %s VALUES (TIMESTAMP '1969-12-31 23:59:59')", table.getName()), + "Timestamp must be between 1970-01-01 00:00:00 and 2105-12-31 23:59:59 in ClickHouse: 1969-12-31 23:59:59"); + assertQueryFails( + format("INSERT INTO %s VALUES (TIMESTAMP '2106-01-01 00:00:00')", table.getName()), + "Timestamp must be between 1970-01-01 00:00:00 and 2105-12-31 23:59:59 in ClickHouse: 2106-01-01 00:00:00"); + assertQueryFails( + format("INSERT INTO %s VALUES (TIMESTAMP '9999-12-31 23:59:59')", table.getName()), + "Timestamp must be between 1970-01-01 00:00:00 and 2105-12-31 23:59:59 in ClickHouse: 9999-12-31 23:59:59"); + } + } + @DataProvider public Object[][] sessionZonesDataProvider() {