From 4f0aecc0470e3ec44d351b49ea6562f41b1a7f64 Mon Sep 17 00:00:00 2001 From: praveenkrishna Date: Fri, 5 May 2023 12:28:27 +0530 Subject: [PATCH 1/5] Minor cleanup in OrcTypeTranslator Use == for enum comparison --- .../main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java index 700fa9499b88..edfe07f869d8 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java @@ -24,6 +24,7 @@ import java.util.Optional; +import static io.trino.orc.metadata.OrcType.OrcTypeKind.TIMESTAMP; import static io.trino.spi.type.TimestampType.createTimestampType; public final class OrcTypeTranslator @@ -32,7 +33,7 @@ private OrcTypeTranslator() {} public static Optional> createCoercer(OrcTypeKind fromOrcType, Type toTrinoType, HiveTimestampPrecision timestampPrecision) { - if (fromOrcType.equals(OrcTypeKind.TIMESTAMP) && toTrinoType instanceof VarcharType varcharType) { + if (fromOrcType == TIMESTAMP && toTrinoType instanceof VarcharType varcharType) { TimestampType timestampType = createTimestampType(timestampPrecision.getPrecision()); if (timestampType.isShort()) { return Optional.of(new ShortTimestampToVarcharCoercer(timestampType, varcharType)); From 0596d327bb72f0bdc313ffc2098d5db952516f32 Mon Sep 17 00:00:00 2001 From: praveenkrishna Date: Fri, 2 Jun 2023 17:03:34 +0530 Subject: [PATCH 2/5] Cleanup in RcFilePageSourceFactory Get type directly from HiveColumnHandle instead of generating them from HiveType. --- .../hive/rcfile/RcFilePageSourceFactory.java | 10 ++------- .../io/trino/plugin/hive/HiveTestUtils.java | 2 +- .../plugin/hive/TestHiveFileFormats.java | 22 +++++++++---------- .../hive/benchmark/StandardFileFormats.java | 5 ++--- 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/rcfile/RcFilePageSourceFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/rcfile/RcFilePageSourceFactory.java index cb73475640e1..aee099959a18 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/rcfile/RcFilePageSourceFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/rcfile/RcFilePageSourceFactory.java @@ -34,7 +34,6 @@ import io.trino.plugin.hive.HiveColumnHandle; import io.trino.plugin.hive.HiveConfig; import io.trino.plugin.hive.HivePageSourceFactory; -import io.trino.plugin.hive.HiveTimestampPrecision; import io.trino.plugin.hive.ReaderColumns; import io.trino.plugin.hive.ReaderPageSource; import io.trino.plugin.hive.acid.AcidTransaction; @@ -45,7 +44,6 @@ import io.trino.spi.connector.EmptyPageSource; import io.trino.spi.predicate.TupleDomain; import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; import org.apache.hadoop.conf.Configuration; import org.joda.time.DateTimeZone; @@ -62,7 +60,6 @@ import static io.trino.plugin.hive.HiveErrorCode.HIVE_BAD_DATA; import static io.trino.plugin.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT; import static io.trino.plugin.hive.HivePageSourceProvider.projectBaseColumns; -import static io.trino.plugin.hive.HiveSessionProperties.getTimestampPrecision; import static io.trino.plugin.hive.ReaderPageSource.noProjectionAdaptation; import static io.trino.plugin.hive.util.HiveClassNames.COLUMNAR_SERDE_CLASS; import static io.trino.plugin.hive.util.HiveClassNames.LAZY_BINARY_COLUMNAR_SERDE_CLASS; @@ -77,15 +74,13 @@ public class RcFilePageSourceFactory { private static final DataSize BUFFER_SIZE = DataSize.of(8, Unit.MEGABYTE); - private final TypeManager typeManager; private final TrinoFileSystemFactory fileSystemFactory; private final FileFormatDataSourceStats stats; private final DateTimeZone timeZone; @Inject - public RcFilePageSourceFactory(TypeManager typeManager, TrinoFileSystemFactory fileSystemFactory, FileFormatDataSourceStats stats, HiveConfig hiveConfig) + public RcFilePageSourceFactory(TrinoFileSystemFactory fileSystemFactory, FileFormatDataSourceStats stats, HiveConfig hiveConfig) { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); this.stats = requireNonNull(stats, "stats is null"); this.timeZone = hiveConfig.getRcfileDateTimeZone(); @@ -168,9 +163,8 @@ else if (deserializerClassName.equals(COLUMNAR_SERDE_CLASS)) { try { ImmutableMap.Builder readColumns = ImmutableMap.builder(); - HiveTimestampPrecision timestampPrecision = getTimestampPrecision(session); for (HiveColumnHandle column : projectedReaderColumns) { - readColumns.put(column.getBaseHiveColumnIndex(), column.getHiveType().getType(typeManager, timestampPrecision)); + readColumns.put(column.getBaseHiveColumnIndex(), column.getType()); } RcFileReader rcFileReader = new RcFileReader( diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveTestUtils.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveTestUtils.java index 5a419fd6d119..b57daea8eacf 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveTestUtils.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveTestUtils.java @@ -206,7 +206,7 @@ public static Set getDefaultHivePageSourceFactories(HdfsE .add(new RegexPageSourceFactory(fileSystemFactory, stats, hiveConfig)) .add(new SimpleTextFilePageSourceFactory(fileSystemFactory, stats, hiveConfig)) .add(new SimpleSequenceFilePageSourceFactory(fileSystemFactory, stats, hiveConfig)) - .add(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, fileSystemFactory, stats, hiveConfig)) + .add(new RcFilePageSourceFactory(fileSystemFactory, stats, hiveConfig)) .add(new OrcPageSourceFactory(new OrcReaderConfig(), fileSystemFactory, stats, hiveConfig)) .add(new ParquetPageSourceFactory(fileSystemFactory, stats, new ParquetReaderConfig(), hiveConfig)) .build(); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java index 7414db5793a3..4904e01a6892 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java @@ -282,7 +282,7 @@ public void testRcTextPageSource(int rowCount, long fileSizePadding) .withColumns(TEST_COLUMNS) .withRowsCount(rowCount) .withFileSizePadding(fileSizePadding) - .isReadableByPageSource(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, FILE_SYSTEM_FACTORY, STATS, new HiveConfig())); + .isReadableByPageSource(new RcFilePageSourceFactory(FILE_SYSTEM_FACTORY, STATS, new HiveConfig())); } @Test(dataProvider = "rowCount") @@ -299,7 +299,7 @@ public void testRcTextOptimizedWriter(int rowCount) .withRowsCount(rowCount) .withFileWriterFactory(new RcFileFileWriterFactory(FILE_SYSTEM_FACTORY, TESTING_TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE)) .isReadableByRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT)) - .isReadableByPageSource(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, FILE_SYSTEM_FACTORY, STATS, new HiveConfig())); + .isReadableByPageSource(new RcFilePageSourceFactory(FILE_SYSTEM_FACTORY, STATS, new HiveConfig())); } @Test(dataProvider = "rowCount") @@ -316,7 +316,7 @@ public void testRcBinaryPageSource(int rowCount) assertThatFileFormat(RCBINARY) .withColumns(testColumns) .withRowsCount(rowCount) - .isReadableByPageSource(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, FILE_SYSTEM_FACTORY, STATS, new HiveConfig())); + .isReadableByPageSource(new RcFilePageSourceFactory(FILE_SYSTEM_FACTORY, STATS, new HiveConfig())); } @Test(dataProvider = "rowCount") @@ -341,7 +341,7 @@ public void testRcBinaryOptimizedWriter(int rowCount) // generic Hive writer corrupts timestamps .withSkipGenericWriterTest() .withFileWriterFactory(new RcFileFileWriterFactory(FILE_SYSTEM_FACTORY, TESTING_TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE)) - .isReadableByPageSource(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, FILE_SYSTEM_FACTORY, STATS, new HiveConfig())) + .isReadableByPageSource(new RcFilePageSourceFactory(FILE_SYSTEM_FACTORY, STATS, new HiveConfig())) .withColumns(testColumnsNoTimestamps) .isReadableByRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT)); } @@ -571,13 +571,13 @@ public void testTruncateVarcharColumn() assertThatFileFormat(RCTEXT) .withWriteColumns(ImmutableList.of(writeColumn)) .withReadColumns(ImmutableList.of(readColumn)) - .isReadableByPageSource(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, FILE_SYSTEM_FACTORY, STATS, new HiveConfig())) + .isReadableByPageSource(new RcFilePageSourceFactory(FILE_SYSTEM_FACTORY, STATS, new HiveConfig())) .isReadableByRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT)); assertThatFileFormat(RCBINARY) .withWriteColumns(ImmutableList.of(writeColumn)) .withReadColumns(ImmutableList.of(readColumn)) - .isReadableByPageSource(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, FILE_SYSTEM_FACTORY, STATS, new HiveConfig())) + .isReadableByPageSource(new RcFilePageSourceFactory(FILE_SYSTEM_FACTORY, STATS, new HiveConfig())) .isReadableByRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT)); assertThatFileFormat(ORC) @@ -804,7 +804,7 @@ public void testRCTextProjectedColumnsPageSource(int rowCount) .withWriteColumns(writeColumns) .withReadColumns(readColumns) .withRowsCount(rowCount) - .isReadableByPageSource(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, FILE_SYSTEM_FACTORY, STATS, new HiveConfig())); + .isReadableByPageSource(new RcFilePageSourceFactory(FILE_SYSTEM_FACTORY, STATS, new HiveConfig())); } @Test(dataProvider = "rowCount") @@ -837,7 +837,7 @@ public void testRCBinaryProjectedColumns(int rowCount) // generic Hive writer corrupts timestamps .withSkipGenericWriterTest() .withFileWriterFactory(new RcFileFileWriterFactory(FILE_SYSTEM_FACTORY, TESTING_TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE)) - .isReadableByPageSource(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, FILE_SYSTEM_FACTORY, STATS, new HiveConfig())); + .isReadableByPageSource(new RcFilePageSourceFactory(FILE_SYSTEM_FACTORY, STATS, new HiveConfig())); } @Test(dataProvider = "rowCount") @@ -867,7 +867,7 @@ public void testRCBinaryProjectedColumnsPageSource(int rowCount) // generic Hive writer corrupts timestamps .withSkipGenericWriterTest() .withFileWriterFactory(new RcFileFileWriterFactory(FILE_SYSTEM_FACTORY, TESTING_TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE)) - .isReadableByPageSource(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, FILE_SYSTEM_FACTORY, STATS, new HiveConfig())); + .isReadableByPageSource(new RcFilePageSourceFactory(FILE_SYSTEM_FACTORY, STATS, new HiveConfig())); } @Test @@ -884,12 +884,12 @@ public void testFailForLongVarcharPartitionColumn() assertThatFileFormat(RCTEXT) .withColumns(columns) - .isFailingForPageSource(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, FILE_SYSTEM_FACTORY, STATS, new HiveConfig()), expectedErrorCode, expectedMessage) + .isFailingForPageSource(new RcFilePageSourceFactory(FILE_SYSTEM_FACTORY, STATS, new HiveConfig()), expectedErrorCode, expectedMessage) .isFailingForRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage); assertThatFileFormat(RCBINARY) .withColumns(columns) - .isFailingForPageSource(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, FILE_SYSTEM_FACTORY, STATS, new HiveConfig()), expectedErrorCode, expectedMessage) + .isFailingForPageSource(new RcFilePageSourceFactory(FILE_SYSTEM_FACTORY, STATS, new HiveConfig()), expectedErrorCode, expectedMessage) .isFailingForRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage); assertThatFileFormat(ORC) diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/benchmark/StandardFileFormats.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/benchmark/StandardFileFormats.java index 49760a125d60..ee450abc9663 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/benchmark/StandardFileFormats.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/benchmark/StandardFileFormats.java @@ -55,7 +55,6 @@ import static io.trino.parquet.writer.ParquetSchemaConverter.HIVE_PARQUET_USE_LEGACY_DECIMAL_ENCODING; import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_FACTORY; import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_STATS; -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; import static org.joda.time.DateTimeZone.UTC; public final class StandardFileFormats @@ -73,7 +72,7 @@ public HiveStorageFormat getFormat() @Override public Optional getHivePageSourceFactory(HdfsEnvironment hdfsEnvironment) { - return Optional.of(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, HDFS_FILE_SYSTEM_FACTORY, new FileFormatDataSourceStats(), new HiveConfig().setRcfileTimeZone("UTC"))); + return Optional.of(new RcFilePageSourceFactory(HDFS_FILE_SYSTEM_FACTORY, new FileFormatDataSourceStats(), new HiveConfig().setRcfileTimeZone("UTC"))); } @Override @@ -104,7 +103,7 @@ public HiveStorageFormat getFormat() @Override public Optional getHivePageSourceFactory(HdfsEnvironment hdfsEnvironment) { - return Optional.of(new RcFilePageSourceFactory(TESTING_TYPE_MANAGER, HDFS_FILE_SYSTEM_FACTORY, new FileFormatDataSourceStats(), new HiveConfig().setRcfileTimeZone("UTC"))); + return Optional.of(new RcFilePageSourceFactory(HDFS_FILE_SYSTEM_FACTORY, new FileFormatDataSourceStats(), new HiveConfig().setRcfileTimeZone("UTC"))); } @Override From 34a105e5939c6b4fb382cba845fcc720997b45a7 Mon Sep 17 00:00:00 2001 From: praveenkrishna Date: Tue, 23 May 2023 13:20:50 +0530 Subject: [PATCH 3/5] For Timestamp to String coercions treat precision as nanoseconds This will be irrespective of the precision configured or specified as session property. --- .../io/trino/plugin/hive/HivePageSource.java | 8 +- .../plugin/hive/HivePageSourceProvider.java | 6 +- .../hive/coercions/TimestampCoercer.java | 23 ----- .../plugin/hive/orc/OrcTypeTranslator.java | 8 +- .../hive/coercions/TestTimestampCoercer.java | 86 +++++++----------- .../product/hive/BaseTestHiveCoercion.java | 80 +++++++++++++++++ .../TestHiveCoercionOnPartitionedTable.java | 89 +++++++++++++++++-- .../TestHiveCoercionOnUnpartitionedTable.java | 29 +++++- 8 files changed, 234 insertions(+), 95 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java index 0300c6d9b2a8..7c0d6f59010b 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java @@ -23,7 +23,6 @@ import io.trino.plugin.hive.coercions.IntegerNumberToVarcharCoercer; import io.trino.plugin.hive.coercions.IntegerNumberUpscaleCoercer; import io.trino.plugin.hive.coercions.TimestampCoercer.LongTimestampToVarcharCoercer; -import io.trino.plugin.hive.coercions.TimestampCoercer.ShortTimestampToVarcharCoercer; import io.trino.plugin.hive.coercions.VarcharCoercer; import io.trino.plugin.hive.coercions.VarcharToIntegerNumberCoercer; import io.trino.plugin.hive.type.Category; @@ -305,7 +304,9 @@ public static Optional> createCoercer(TypeManager typeMan return Optional.empty(); } - Type fromType = fromHiveType.getType(typeManager, timestampPrecision); + // Hive treats TIMESTAMP with NANOSECONDS precision and when we try to coerce from a timestamp column, + // we read it as TIMESTAMP(9) column and coerce accordingly. + Type fromType = fromHiveType.getType(typeManager, HiveTimestampPrecision.NANOSECONDS); Type toType = toHiveType.getType(typeManager, timestampPrecision); if (toType instanceof VarcharType toVarcharType && (fromHiveType.equals(HIVE_BYTE) || fromHiveType.equals(HIVE_SHORT) || fromHiveType.equals(HIVE_INT) || fromHiveType.equals(HIVE_LONG))) { @@ -360,9 +361,6 @@ public static Optional> createCoercer(TypeManager typeMan return Optional.of(createRealToDecimalCoercer(toDecimalType)); } if (fromType instanceof TimestampType timestampType && toType instanceof VarcharType varcharType) { - if (timestampType.isShort()) { - return Optional.of(new ShortTimestampToVarcharCoercer(timestampType, varcharType)); - } return Optional.of(new LongTimestampToVarcharCoercer(timestampType, varcharType)); } if ((fromType instanceof ArrayType) && (toType instanceof ArrayType)) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSourceProvider.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSourceProvider.java index 4b12227bf59a..0e6a6c1fee9d 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSourceProvider.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSourceProvider.java @@ -560,14 +560,16 @@ public static List toColumnHandles(List regular projectedColumn.getDereferenceIndices(), projectedColumn.getDereferenceNames(), fromHiveType, - fromHiveType.getType(typeManager)); + // Hive treats TIMESTAMP with NANOSECONDS precision and when we try to coerce from a timestamp column, + // we read it as TIMESTAMP(9) column and coerce accordingly. + fromHiveType.getType(typeManager, HiveTimestampPrecision.NANOSECONDS)); }); return new HiveColumnHandle( columnHandle.getBaseColumnName(), columnHandle.getBaseHiveColumnIndex(), fromHiveTypeBase, - fromHiveTypeBase.getType(typeManager), + fromHiveTypeBase.getType(typeManager, HiveTimestampPrecision.NANOSECONDS), newColumnProjectionInfo, columnHandle.getColumnType(), columnHandle.getComment()); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/TimestampCoercer.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/TimestampCoercer.java index 2e15793365e5..b44f5295f100 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/TimestampCoercer.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/TimestampCoercer.java @@ -48,29 +48,6 @@ public final class TimestampCoercer private TimestampCoercer() {} - public static class ShortTimestampToVarcharCoercer - extends TypeCoercer - { - public ShortTimestampToVarcharCoercer(TimestampType fromType, VarcharType toType) - { - super(fromType, toType); - } - - @Override - protected void applyCoercedValue(BlockBuilder blockBuilder, Block block, int position) - { - long epochMicros = fromType.getLong(block, position); - long epochSecond = floorDiv(epochMicros, MICROSECONDS_PER_SECOND); - int nanoFraction = floorMod(epochMicros, MICROSECONDS_PER_SECOND) * NANOSECONDS_PER_MICROSECOND; - toType.writeSlice( - blockBuilder, - truncateToLength( - Slices.utf8Slice( - LOCAL_DATE_TIME.format(LocalDateTime.ofEpochSecond(epochSecond, nanoFraction, UTC))), - toType)); - } - } - public static class LongTimestampToVarcharCoercer extends TypeCoercer { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java index edfe07f869d8..f40fd6b1d91a 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java @@ -16,7 +16,6 @@ import io.trino.orc.metadata.OrcType.OrcTypeKind; import io.trino.plugin.hive.HiveTimestampPrecision; import io.trino.plugin.hive.coercions.TimestampCoercer.LongTimestampToVarcharCoercer; -import io.trino.plugin.hive.coercions.TimestampCoercer.ShortTimestampToVarcharCoercer; import io.trino.plugin.hive.coercions.TypeCoercer; import io.trino.spi.type.TimestampType; import io.trino.spi.type.Type; @@ -34,10 +33,9 @@ private OrcTypeTranslator() {} public static Optional> createCoercer(OrcTypeKind fromOrcType, Type toTrinoType, HiveTimestampPrecision timestampPrecision) { if (fromOrcType == TIMESTAMP && toTrinoType instanceof VarcharType varcharType) { - TimestampType timestampType = createTimestampType(timestampPrecision.getPrecision()); - if (timestampType.isShort()) { - return Optional.of(new ShortTimestampToVarcharCoercer(timestampType, varcharType)); - } + // Hive treats TIMESTAMP with NANOSECONDS precision and when we try to coerce from a timestamp column, + // we read it as TIMESTAMP(9) column and coerce accordingly. + TimestampType timestampType = createTimestampType(HiveTimestampPrecision.NANOSECONDS.getPrecision()); return Optional.of(new LongTimestampToVarcharCoercer(timestampType, varcharType)); } return Optional.empty(); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestTimestampCoercer.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestTimestampCoercer.java index 91bf91fff433..ed65927c80e2 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestTimestampCoercer.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestTimestampCoercer.java @@ -27,12 +27,10 @@ import java.time.LocalDateTime; import static io.trino.plugin.hive.HivePageSource.createCoercer; -import static io.trino.plugin.hive.HiveTimestampPrecision.MICROSECONDS; import static io.trino.plugin.hive.HiveTimestampPrecision.NANOSECONDS; import static io.trino.plugin.hive.HiveType.toHiveType; import static io.trino.spi.predicate.Utils.blockToNativeValue; import static io.trino.spi.predicate.Utils.nativeValueToBlock; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MICROS; import static io.trino.spi.type.TimestampType.TIMESTAMP_PICOS; import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; import static io.trino.spi.type.VarcharType.createVarcharType; @@ -44,15 +42,7 @@ public class TestTimestampCoercer { @Test(dataProvider = "timestampValuesProvider") - public void testShortTimestampToVarchar(String timestampValue, String hiveTimestampValue) - { - LocalDateTime localDateTime = LocalDateTime.parse(timestampValue); - SqlTimestamp timestamp = SqlTimestamp.fromSeconds(TIMESTAMP_MICROS.getPrecision(), localDateTime.toEpochSecond(UTC), localDateTime.get(NANO_OF_SECOND)); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createUnboundedVarcharType(), hiveTimestampValue); - } - - @Test(dataProvider = "timestampValuesProvider") - public void testLongTimestampToVarchar(String timestampValue, String hiveTimestampValue) + public void testTimestampToVarchar(String timestampValue, String hiveTimestampValue) { LocalDateTime localDateTime = LocalDateTime.parse(timestampValue); SqlTimestamp timestamp = SqlTimestamp.fromSeconds(TIMESTAMP_PICOS.getPrecision(), localDateTime.toEpochSecond(UTC), localDateTime.get(NANO_OF_SECOND)); @@ -60,47 +50,40 @@ public void testLongTimestampToVarchar(String timestampValue, String hiveTimesta } @Test - public void testShortTimestampToSmallerVarchar() - { - LocalDateTime localDateTime = LocalDateTime.parse("2023-04-11T05:16:12.345678"); - SqlTimestamp timestamp = SqlTimestamp.fromSeconds(TIMESTAMP_MICROS.getPrecision(), localDateTime.toEpochSecond(UTC), localDateTime.get(NANO_OF_SECOND)); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(1), "2"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(2), "20"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(3), "202"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(4), "2023"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(5), "2023-"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(6), "2023-0"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(7), "2023-04"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(8), "2023-04-"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(9), "2023-04-1"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(10), "2023-04-11"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(11), "2023-04-11 "); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(12), "2023-04-11 0"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(13), "2023-04-11 05"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(14), "2023-04-11 05:"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(15), "2023-04-11 05:1"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(16), "2023-04-11 05:16"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(17), "2023-04-11 05:16:"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(18), "2023-04-11 05:16:1"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(19), "2023-04-11 05:16:12"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(20), "2023-04-11 05:16:12."); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(21), "2023-04-11 05:16:12.3"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(22), "2023-04-11 05:16:12.34"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(23), "2023-04-11 05:16:12.345"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(24), "2023-04-11 05:16:12.3456"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(25), "2023-04-11 05:16:12.34567"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(26), "2023-04-11 05:16:12.345678"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(27), "2023-04-11 05:16:12.345678"); - } - - @Test - public void testLongTimestampToSmallerVarchar() + public void testTimestampToSmallerVarchar() { LocalDateTime localDateTime = LocalDateTime.parse("2023-04-11T05:16:12.345678876"); SqlTimestamp timestamp = SqlTimestamp.fromSeconds(TIMESTAMP_PICOS.getPrecision(), localDateTime.toEpochSecond(UTC), localDateTime.get(NANO_OF_SECOND)); - assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, new LongTimestamp(timestamp.getEpochMicros(), timestamp.getPicosOfMicros()), createVarcharType(27), "2023-04-11 05:16:12.3456788"); - assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, new LongTimestamp(timestamp.getEpochMicros(), timestamp.getPicosOfMicros()), createVarcharType(28), "2023-04-11 05:16:12.34567887"); - assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, new LongTimestamp(timestamp.getEpochMicros(), timestamp.getPicosOfMicros()), createVarcharType(29), "2023-04-11 05:16:12.345678876"); + LongTimestamp longTimestamp = new LongTimestamp(timestamp.getEpochMicros(), timestamp.getPicosOfMicros()); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(1), "2"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(2), "20"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(3), "202"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(4), "2023"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(5), "2023-"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(6), "2023-0"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(7), "2023-04"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(8), "2023-04-"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(9), "2023-04-1"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(10), "2023-04-11"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(11), "2023-04-11 "); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(12), "2023-04-11 0"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(13), "2023-04-11 05"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(14), "2023-04-11 05:"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(15), "2023-04-11 05:1"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(16), "2023-04-11 05:16"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(17), "2023-04-11 05:16:"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(18), "2023-04-11 05:16:1"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(19), "2023-04-11 05:16:12"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(20), "2023-04-11 05:16:12."); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(21), "2023-04-11 05:16:12.3"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(22), "2023-04-11 05:16:12.34"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(23), "2023-04-11 05:16:12.345"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(24), "2023-04-11 05:16:12.3456"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(25), "2023-04-11 05:16:12.34567"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(26), "2023-04-11 05:16:12.345678"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(27), "2023-04-11 05:16:12.3456788"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(28), "2023-04-11 05:16:12.34567887"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(29), "2023-04-11 05:16:12.345678876"); } @DataProvider @@ -129,11 +112,6 @@ public Object[][] timestampValuesProvider() }; } - public static void assertShortTimestampToVarcharCoercions(TimestampType fromType, Long valueToBeCoerced, VarcharType toType, String expectedValue) - { - assertCoercions(fromType, valueToBeCoerced, toType, Slices.utf8Slice(expectedValue), MICROSECONDS); - } - public static void assertLongTimestampToVarcharCoercions(TimestampType fromType, LongTimestamp valueToBeCoerced, VarcharType toType, String expectedValue) { assertCoercions(fromType, valueToBeCoerced, toType, Slices.utf8Slice(expectedValue), NANOSECONDS); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java index 606c3c9c1b24..b18948d74409 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java @@ -16,7 +16,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import io.trino.jdbc.TrinoArray; +import io.trino.plugin.hive.HiveTimestampPrecision; import io.trino.tempto.assertions.QueryAssert.Row; import io.trino.tempto.fulfillment.table.MutableTablesState; import io.trino.tempto.fulfillment.table.TableDefinition; @@ -28,6 +30,7 @@ import java.math.BigDecimal; import java.sql.JDBCType; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -41,10 +44,12 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; +import static io.trino.plugin.hive.HiveTimestampPrecision.NANOSECONDS; import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.tempto.assertions.QueryAssert.assertThat; import static io.trino.tempto.context.ThreadLocalTestContextHolder.testContext; import static io.trino.tempto.fulfillment.table.TableHandle.tableHandle; +import static io.trino.tests.product.utils.JdbcDriverUtils.setSessionProperty; import static io.trino.tests.product.utils.QueryExecutors.onHive; import static io.trino.tests.product.utils.QueryExecutors.onTrino; import static java.lang.String.format; @@ -60,6 +65,7 @@ import static java.sql.JDBCType.SMALLINT; import static java.sql.JDBCType.STRUCT; import static java.sql.JDBCType.VARCHAR; +import static java.util.Collections.nCopies; import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; @@ -364,6 +370,69 @@ else if (getHiveVersionMajor() == 3 && isFormat.test("orc")) { .buildOrThrow(); } + protected void doTestHiveCoercionWithDifferentTimestampPrecision(HiveTableDefinition tableDefinition) + { + String tableName = mutableTableInstanceOf(tableDefinition).getNameInDatabase(); + + // Insert all the data with nanoseconds precision + setHiveTimestampPrecision(NANOSECONDS); + onTrino().executeQuery( + """ + INSERT INTO %s VALUES + (TIMESTAMP '2121-07-15 15:30:12.123499', 1), + (TIMESTAMP '2121-07-15 15:30:12.123500', 1), + (TIMESTAMP '2121-07-15 15:30:12.123501', 1), + (TIMESTAMP '2121-07-15 15:30:12.123499999', 1), + (TIMESTAMP '2121-07-15 15:30:12.123500000', 1), + (TIMESTAMP '2121-07-15 15:30:12.123500001', 1) + """.formatted(tableName)); + + onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN timestamp_to_varchar timestamp_to_varchar STRING", tableName)); + + for (HiveTimestampPrecision hiveTimestampPrecision : HiveTimestampPrecision.values()) { + setHiveTimestampPrecision(hiveTimestampPrecision); + assertThat(onTrino().executeQuery("SHOW COLUMNS FROM " + tableName).project(1, 2)).containsExactlyInOrder( + row("timestamp_to_varchar", "varchar"), + row("id", "bigint")); + + List allColumns = ImmutableList.of( + "timestamp_to_varchar", + "id"); + + // For Trino, remove unsupported columns + List trinoReadColumns = removeUnsupportedColumnsForTrino(allColumns, tableName); + Map> expectedTinoResults = Maps.filterKeys( + expectedRowsForEngineProvider(Engine.TRINO, hiveTimestampPrecision), + trinoReadColumns::contains); + + String trinoReadQuery = format("SELECT %s FROM %s", String.join(", ", trinoReadColumns), tableName); + assertQueryResults(Engine.TRINO, trinoReadQuery, expectedTinoResults, trinoReadColumns, 6, tableName); + + List hiveReadColumns = removeUnsupportedColumnsForHive(allColumns, tableName); + Map> expectedHiveResults = Maps.filterKeys( + expectedRowsForEngineProvider(Engine.HIVE, hiveTimestampPrecision), + hiveReadColumns::contains); + + String hiveSelectQuery = format("SELECT %s FROM %s", String.join(", ", hiveReadColumns), tableName); + assertQueryResults(Engine.HIVE, hiveSelectQuery, expectedHiveResults, hiveReadColumns, 6, tableName); + } + } + + protected Map> expectedRowsForEngineProvider(Engine engine, HiveTimestampPrecision hiveTimestampPrecision) + { + ImmutableMap.Builder> rowBuilder = ImmutableMap.>builder() + .put("timestamp_to_varchar", ImmutableList.of( + "2121-07-15 15:30:12.123499", + "2121-07-15 15:30:12.1235", + "2121-07-15 15:30:12.123501", + "2121-07-15 15:30:12.123499999", + "2121-07-15 15:30:12.1235", + "2121-07-15 15:30:12.123500001")) + .put("id", nCopies(6, 1)); + + return rowBuilder.buildOrThrow(); + } + protected List removeUnsupportedColumnsForHive(List columns, String tableName) { // TODO: assert exceptions being thrown for each column @@ -636,6 +705,7 @@ private void assertColumnTypes( .put("timestamp_to_string", VARCHAR) .put("timestamp_to_bounded_varchar", VARCHAR) .put("timestamp_to_smaller_varchar", VARCHAR) + .put("timestamp_to_varchar", VARCHAR) .buildOrThrow(); assertThat(queryResult) @@ -763,4 +833,14 @@ private static QueryResult execute(Engine engine, String sql, QueryExecutor.Quer { return engine.queryExecutor().executeQuery(sql, params); } + + private static void setHiveTimestampPrecision(HiveTimestampPrecision hiveTimestampPrecision) + { + try { + setSessionProperty(onTrino().getConnection(), "hive.timestamp_precision", hiveTimestampPrecision.name()); + } + catch (SQLException e) { + throw new RuntimeException(e); + } + } } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnPartitionedTable.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnPartitionedTable.java index 2004b547219d..7ca21f51725a 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnPartitionedTable.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnPartitionedTable.java @@ -24,6 +24,7 @@ import java.util.Optional; +import static io.trino.tempto.Requirements.compose; import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.tempto.assertions.QueryAssert.assertThat; import static io.trino.tempto.fulfillment.table.MutableTableRequirement.State.CREATED; @@ -43,10 +44,18 @@ public class TestHiveCoercionOnPartitionedTable .setNoData() .build(); + public static final HiveTableDefinition HIVE_TIMESTAMP_COERCION_TEXTFILE = tableDefinitionForTimestampCoercionBuilder("TEXTFILE", Optional.empty(), Optional.of("DELIMITED FIELDS TERMINATED BY '|'")) + .setNoData() + .build(); + public static final HiveTableDefinition HIVE_COERCION_PARQUET = tableDefinitionBuilder("PARQUET", Optional.empty(), Optional.empty()) .setNoData() .build(); + public static final HiveTableDefinition HIVE_TIMESTAMP_COERCION_PARQUET = tableDefinitionForTimestampCoercionBuilder("PARQUET", Optional.empty(), Optional.empty()) + .setNoData() + .build(); + public static final HiveTableDefinition HIVE_COERCION_AVRO = avroTableDefinitionBuilder() .setNoData() .build(); @@ -55,14 +64,26 @@ public class TestHiveCoercionOnPartitionedTable .setNoData() .build(); + public static final HiveTableDefinition HIVE_TIMESTAMP_COERCION_ORC = tableDefinitionForTimestampCoercionBuilder("ORC", Optional.empty(), Optional.empty()) + .setNoData() + .build(); + public static final HiveTableDefinition HIVE_COERCION_RCTEXT = tableDefinitionBuilder("RCFILE", Optional.of("RCTEXT"), Optional.of("SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'")) .setNoData() .build(); + public static final HiveTableDefinition HIVE_TIMESTAMP_COERCION_RCTEXT = tableDefinitionForTimestampCoercionBuilder("RCFILE", Optional.of("RCTEXT"), Optional.of("SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'")) + .setNoData() + .build(); + public static final HiveTableDefinition HIVE_COERCION_RCBINARY = tableDefinitionBuilder("RCFILE", Optional.of("RCBINARY"), Optional.of("SERDE 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe'")) .setNoData() .build(); + public static final HiveTableDefinition HIVE_TIMESTAMP_COERCION_RCBINARY = tableDefinitionForTimestampCoercionBuilder("RCFILE", Optional.of("RCBINARY"), Optional.of("SERDE 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe'")) + .setNoData() + .build(); + private static HiveTableDefinition.HiveTableDefinitionBuilder tableDefinitionBuilder(String fileFormat, Optional recommendTableName, Optional rowFormat) { String tableName = format("%s_hive_coercion", recommendTableName.orElse(fileFormat).toLowerCase(ENGLISH)); @@ -108,6 +129,19 @@ private static HiveTableDefinition.HiveTableDefinitionBuilder tableDefinitionBui "STORED AS " + fileFormat); } + private static HiveTableDefinition.HiveTableDefinitionBuilder tableDefinitionForTimestampCoercionBuilder(String fileFormat, Optional recommendTableName, Optional rowFormat) + { + String tableName = format("%s_hive_timestamp_coercion", recommendTableName.orElse(fileFormat).toLowerCase(ENGLISH)); + return HiveTableDefinition.builder(tableName) + .setCreateTableDDLTemplate("" + + "CREATE TABLE %NAME%(" + + " timestamp_to_varchar TIMESTAMP" + + ") " + + "PARTITIONED BY (id BIGINT) " + + rowFormat.map(s -> format("ROW FORMAT %s ", s)).orElse("") + + "STORED AS " + fileFormat); + } + private static HiveTableDefinition.HiveTableDefinitionBuilder avroTableDefinitionBuilder() { return HiveTableDefinition.builder("avro_hive_coercion") @@ -126,7 +160,9 @@ public static final class TextRequirements @Override public Requirement getRequirements(Configuration configuration) { - return MutableTableRequirement.builder(HIVE_COERCION_TEXTFILE).withState(CREATED).build(); + return compose( + MutableTableRequirement.builder(HIVE_COERCION_TEXTFILE).withState(CREATED).build(), + MutableTableRequirement.builder(HIVE_TIMESTAMP_COERCION_TEXTFILE).withState(CREATED).build()); } } @@ -136,7 +172,9 @@ public static final class OrcRequirements @Override public Requirement getRequirements(Configuration configuration) { - return MutableTableRequirement.builder(HIVE_COERCION_ORC).withState(CREATED).build(); + return compose( + MutableTableRequirement.builder(HIVE_COERCION_ORC).withState(CREATED).build(), + MutableTableRequirement.builder(HIVE_TIMESTAMP_COERCION_ORC).withState(CREATED).build()); } } @@ -146,7 +184,9 @@ public static final class RcTextRequirements @Override public Requirement getRequirements(Configuration configuration) { - return MutableTableRequirement.builder(HIVE_COERCION_RCTEXT).withState(CREATED).build(); + return compose( + MutableTableRequirement.builder(HIVE_COERCION_RCTEXT).withState(CREATED).build(), + MutableTableRequirement.builder(HIVE_TIMESTAMP_COERCION_RCTEXT).withState(CREATED).build()); } } @@ -156,7 +196,9 @@ public static final class RcBinaryRequirements @Override public Requirement getRequirements(Configuration configuration) { - return MutableTableRequirement.builder(HIVE_COERCION_RCBINARY).withState(CREATED).build(); + return compose( + MutableTableRequirement.builder(HIVE_COERCION_RCBINARY).withState(CREATED).build(), + MutableTableRequirement.builder(HIVE_TIMESTAMP_COERCION_RCBINARY).withState(CREATED).build()); } } @@ -166,7 +208,9 @@ public static final class ParquetRequirements @Override public Requirement getRequirements(Configuration configuration) { - return MutableTableRequirement.builder(HIVE_COERCION_PARQUET).withState(CREATED).build(); + return compose( + MutableTableRequirement.builder(HIVE_COERCION_PARQUET).withState(CREATED).build(), + MutableTableRequirement.builder(HIVE_TIMESTAMP_COERCION_PARQUET).withState(CREATED).build()); } } @@ -187,6 +231,13 @@ public void testHiveCoercionTextFile() doTestHiveCoercion(HIVE_COERCION_TEXTFILE); } + @Requires(TextRequirements.class) + @Test(groups = {HIVE_COERCION, JDBC}) + public void testHiveTimestampCoercionTextFile() + { + doTestHiveCoercionWithDifferentTimestampPrecision(HIVE_TIMESTAMP_COERCION_TEXTFILE); + } + @Requires(OrcRequirements.class) @Test(groups = {HIVE_COERCION, JDBC}) public void testHiveCoercionOrc() @@ -194,6 +245,13 @@ public void testHiveCoercionOrc() doTestHiveCoercion(HIVE_COERCION_ORC); } + @Requires(OrcRequirements.class) + @Test(groups = {HIVE_COERCION, JDBC}) + public void testHiveTimestampCoercionOrc() + { + doTestHiveCoercionWithDifferentTimestampPrecision(HIVE_TIMESTAMP_COERCION_ORC); + } + @Requires(RcTextRequirements.class) @Test(groups = {HIVE_COERCION, JDBC}) public void testHiveCoercionRcText() @@ -201,6 +259,13 @@ public void testHiveCoercionRcText() doTestHiveCoercion(HIVE_COERCION_RCTEXT); } + @Requires(RcTextRequirements.class) + @Test(groups = {HIVE_COERCION, JDBC}) + public void testHiveTimestampCoercionRcText() + { + doTestHiveCoercionWithDifferentTimestampPrecision(HIVE_TIMESTAMP_COERCION_RCTEXT); + } + @Requires(RcBinaryRequirements.class) @Test(groups = {HIVE_COERCION, JDBC}) public void testHiveCoercionRcBinary() @@ -208,6 +273,13 @@ public void testHiveCoercionRcBinary() doTestHiveCoercion(HIVE_COERCION_RCBINARY); } + @Requires(RcBinaryRequirements.class) + @Test(groups = {HIVE_COERCION, JDBC}) + public void testHiveTimestampCoercionRcBinary() + { + doTestHiveCoercionWithDifferentTimestampPrecision(HIVE_TIMESTAMP_COERCION_RCBINARY); + } + @Requires(ParquetRequirements.class) @Test(groups = {HIVE_COERCION, JDBC}) public void testHiveCoercionParquet() @@ -215,6 +287,13 @@ public void testHiveCoercionParquet() doTestHiveCoercion(HIVE_COERCION_PARQUET); } + @Requires(ParquetRequirements.class) + @Test(groups = {HIVE_COERCION, JDBC}) + public void testHiveTimestampCoercionParquet() + { + doTestHiveCoercionWithDifferentTimestampPrecision(HIVE_TIMESTAMP_COERCION_PARQUET); + } + @Requires(AvroRequirements.class) @Test(groups = {HIVE_COERCION, JDBC}) public void testHiveCoercionAvro() diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnUnpartitionedTable.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnUnpartitionedTable.java index a430a1fd2ff6..15e21fb58c5e 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnUnpartitionedTable.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnUnpartitionedTable.java @@ -24,6 +24,7 @@ import java.util.Map; +import static io.trino.tempto.Requirements.compose; import static io.trino.tempto.fulfillment.table.MutableTableRequirement.State.CREATED; import static io.trino.tests.product.TestGroups.HIVE_COERCION; import static io.trino.tests.product.TestGroups.JDBC; @@ -37,6 +38,10 @@ public class TestHiveCoercionOnUnpartitionedTable .setNoData() .build(); + public static final HiveTableDefinition HIVE_TIMESTAMP_COERCION_ORC = tableDefinitionForTimestampCoercionBuilder("ORC") + .setNoData() + .build(); + private static HiveTableDefinition.HiveTableDefinitionBuilder tableDefinitionBuilder(String fileFormat) { String tableName = format("%s_hive_coercion_unpartitioned", fileFormat.toLowerCase(ENGLISH)); @@ -79,13 +84,26 @@ char_to_smaller_char CHAR(3), STORED AS\s""" + fileFormat); } + private static HiveTableDefinition.HiveTableDefinitionBuilder tableDefinitionForTimestampCoercionBuilder(String fileFormat) + { + String tableName = format("%s_hive_timestamp_coercion_unpartitioned", fileFormat.toLowerCase(ENGLISH)); + return HiveTableDefinition.builder(tableName) + .setCreateTableDDLTemplate(""" + CREATE TABLE %NAME%( + timestamp_to_varchar TIMESTAMP, + id BIGINT) + STORED AS\s""" + fileFormat); + } + public static final class OrcRequirements implements RequirementsProvider { @Override public Requirement getRequirements(Configuration configuration) { - return MutableTableRequirement.builder(HIVE_COERCION_ORC).withState(CREATED).build(); + return compose( + MutableTableRequirement.builder(HIVE_COERCION_ORC).withState(CREATED).build(), + MutableTableRequirement.builder(HIVE_TIMESTAMP_COERCION_ORC).withState(CREATED).build()); } } @@ -96,11 +114,20 @@ public void testHiveCoercionOrc() doTestHiveCoercion(HIVE_COERCION_ORC); } + @Requires(OrcRequirements.class) + @Test(groups = {HIVE_COERCION, JDBC}) + public void testHiveTimestampCoercion() + { + doTestHiveCoercionWithDifferentTimestampPrecision(HIVE_TIMESTAMP_COERCION_ORC); + } + @Override protected Map expectedExceptionsWithTrinoContext() { // TODO: These expected failures should be fixed. return ImmutableMap.builder() + // Expected failures from BaseTestHiveCoercion + .putAll(super.expectedExceptionsWithTrinoContext()) // ORC .put(columnContext("orc", "row_to_row"), "Cannot read SQL type 'smallint' from ORC stream '.row_to_row.ti2si' of type BYTE") .put(columnContext("orc", "list_to_list"), "Cannot read SQL type 'integer' from ORC stream '.list_to_list.item.ti2int' of type BYTE") From c64d36b23460ca8f8d7103f7d18f5de23f49c8e7 Mon Sep 17 00:00:00 2001 From: praveenkrishna Date: Mon, 5 Jun 2023 16:55:12 +0530 Subject: [PATCH 4/5] fixup! For Timestamp to String coercions treat precision as nanoseconds This will be irrespective of the precision configured or specified as session property. --- .../io/trino/plugin/hive/HivePageSource.java | 7 ++++--- .../trino/plugin/hive/orc/OrcTypeTranslator.java | 6 ++---- .../tests/product/hive/BaseTestHiveCoercion.java | 16 +++++++++++++--- .../TestHiveCoercionOnUnpartitionedTable.java | 2 -- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java index 7c0d6f59010b..69fab87eb809 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java @@ -95,6 +95,7 @@ import static io.trino.spi.block.ColumnarRow.toColumnarRow; import static io.trino.spi.type.DoubleType.DOUBLE; import static io.trino.spi.type.RealType.REAL; +import static io.trino.spi.type.TimestampType.TIMESTAMP_NANOS; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -306,7 +307,7 @@ public static Optional> createCoercer(TypeManager typeMan // Hive treats TIMESTAMP with NANOSECONDS precision and when we try to coerce from a timestamp column, // we read it as TIMESTAMP(9) column and coerce accordingly. - Type fromType = fromHiveType.getType(typeManager, HiveTimestampPrecision.NANOSECONDS); + Type fromType = fromHiveType.getType(typeManager, timestampPrecision); Type toType = toHiveType.getType(typeManager, timestampPrecision); if (toType instanceof VarcharType toVarcharType && (fromHiveType.equals(HIVE_BYTE) || fromHiveType.equals(HIVE_SHORT) || fromHiveType.equals(HIVE_INT) || fromHiveType.equals(HIVE_LONG))) { @@ -360,8 +361,8 @@ public static Optional> createCoercer(TypeManager typeMan if (fromType == REAL && toType instanceof DecimalType toDecimalType) { return Optional.of(createRealToDecimalCoercer(toDecimalType)); } - if (fromType instanceof TimestampType timestampType && toType instanceof VarcharType varcharType) { - return Optional.of(new LongTimestampToVarcharCoercer(timestampType, varcharType)); + if (fromType instanceof TimestampType && toType instanceof VarcharType varcharType) { + return Optional.of(new LongTimestampToVarcharCoercer(TIMESTAMP_NANOS, varcharType)); } if ((fromType instanceof ArrayType) && (toType instanceof ArrayType)) { return Optional.of(new ListCoercer(typeManager, fromHiveType, toHiveType, timestampPrecision)); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java index f40fd6b1d91a..2491c6a75067 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java @@ -17,14 +17,13 @@ import io.trino.plugin.hive.HiveTimestampPrecision; import io.trino.plugin.hive.coercions.TimestampCoercer.LongTimestampToVarcharCoercer; import io.trino.plugin.hive.coercions.TypeCoercer; -import io.trino.spi.type.TimestampType; import io.trino.spi.type.Type; import io.trino.spi.type.VarcharType; import java.util.Optional; import static io.trino.orc.metadata.OrcType.OrcTypeKind.TIMESTAMP; -import static io.trino.spi.type.TimestampType.createTimestampType; +import static io.trino.spi.type.TimestampType.TIMESTAMP_NANOS; public final class OrcTypeTranslator { @@ -35,8 +34,7 @@ private OrcTypeTranslator() {} if (fromOrcType == TIMESTAMP && toTrinoType instanceof VarcharType varcharType) { // Hive treats TIMESTAMP with NANOSECONDS precision and when we try to coerce from a timestamp column, // we read it as TIMESTAMP(9) column and coerce accordingly. - TimestampType timestampType = createTimestampType(HiveTimestampPrecision.NANOSECONDS.getPrecision()); - return Optional.of(new LongTimestampToVarcharCoercer(timestampType, varcharType)); + return Optional.of(new LongTimestampToVarcharCoercer(TIMESTAMP_NANOS, varcharType)); } return Optional.empty(); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java index b18948d74409..8282b7a8f4e0 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java @@ -399,10 +399,20 @@ protected void doTestHiveCoercionWithDifferentTimestampPrecision(HiveTableDefini "timestamp_to_varchar", "id"); + Map> expectedRows = ImmutableMap.of( + "timestamp_to_varchar", ImmutableList.of( + "2121-07-15 15:30:12.123499", + "2121-07-15 15:30:12.1235", + "2121-07-15 15:30:12.123501", + "2121-07-15 15:30:12.123499999", + "2121-07-15 15:30:12.1235", + "2121-07-15 15:30:12.123500001"), + "id", nCopies(6, 1)); + // For Trino, remove unsupported columns List trinoReadColumns = removeUnsupportedColumnsForTrino(allColumns, tableName); Map> expectedTinoResults = Maps.filterKeys( - expectedRowsForEngineProvider(Engine.TRINO, hiveTimestampPrecision), + expectedRows, trinoReadColumns::contains); String trinoReadQuery = format("SELECT %s FROM %s", String.join(", ", trinoReadColumns), tableName); @@ -410,7 +420,7 @@ protected void doTestHiveCoercionWithDifferentTimestampPrecision(HiveTableDefini List hiveReadColumns = removeUnsupportedColumnsForHive(allColumns, tableName); Map> expectedHiveResults = Maps.filterKeys( - expectedRowsForEngineProvider(Engine.HIVE, hiveTimestampPrecision), + expectedRows, hiveReadColumns::contains); String hiveSelectQuery = format("SELECT %s FROM %s", String.join(", ", hiveReadColumns), tableName); @@ -418,7 +428,7 @@ protected void doTestHiveCoercionWithDifferentTimestampPrecision(HiveTableDefini } } - protected Map> expectedRowsForEngineProvider(Engine engine, HiveTimestampPrecision hiveTimestampPrecision) + protected Map> expectedRowsForEngineProvider() { ImmutableMap.Builder> rowBuilder = ImmutableMap.>builder() .put("timestamp_to_varchar", ImmutableList.of( diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnUnpartitionedTable.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnUnpartitionedTable.java index 15e21fb58c5e..5d52c8d25b05 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnUnpartitionedTable.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnUnpartitionedTable.java @@ -126,8 +126,6 @@ protected Map expectedExceptionsWithTrinoContext() { // TODO: These expected failures should be fixed. return ImmutableMap.builder() - // Expected failures from BaseTestHiveCoercion - .putAll(super.expectedExceptionsWithTrinoContext()) // ORC .put(columnContext("orc", "row_to_row"), "Cannot read SQL type 'smallint' from ORC stream '.row_to_row.ti2si' of type BYTE") .put(columnContext("orc", "list_to_list"), "Cannot read SQL type 'integer' from ORC stream '.list_to_list.item.ti2int' of type BYTE") From 030c40c69df1c34f7c6d4f9de5a5689b37c9a46d Mon Sep 17 00:00:00 2001 From: praveenkrishna Date: Tue, 23 May 2023 14:57:42 +0530 Subject: [PATCH 5/5] Restrict timestamp to varchar coercion for historical dates Hive 2.+ and Hive 3.+ uses `java.sql.Timestamp#toString` for coercing Timestamp to Varchar types. `java.sql.Timestamp#toString` doesn't capture the historical dates correctly --- .../io/trino/plugin/hive/HiveErrorCode.java | 1 + .../plugin/hive/coercions/TimestampCoercer.java | 9 +++++++++ .../hive/coercions/TestTimestampCoercer.java | 17 +++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveErrorCode.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveErrorCode.java index c628d970b120..2018867e6381 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveErrorCode.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveErrorCode.java @@ -67,6 +67,7 @@ public enum HiveErrorCode HIVE_TABLE_LOCK_NOT_ACQUIRED(40, EXTERNAL), HIVE_VIEW_TRANSLATION_ERROR(41, EXTERNAL), HIVE_PARTITION_NOT_FOUND(42, USER_ERROR), + HIVE_INVALID_TIMESTAMP_COERCION(43, EXTERNAL), /**/; private final ErrorCode errorCode; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/TimestampCoercer.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/TimestampCoercer.java index b44f5295f100..0f9ab56c0dc6 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/TimestampCoercer.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/TimestampCoercer.java @@ -14,6 +14,7 @@ package io.trino.plugin.hive.coercions; import io.airlift.slice.Slices; +import io.trino.spi.TrinoException; import io.trino.spi.block.Block; import io.trino.spi.block.BlockBuilder; import io.trino.spi.type.LongTimestamp; @@ -25,9 +26,11 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; +import static io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_TIMESTAMP_COERCION; import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_SECOND; import static io.trino.spi.type.Timestamps.NANOSECONDS_PER_MICROSECOND; import static io.trino.spi.type.Timestamps.PICOSECONDS_PER_NANOSECOND; +import static io.trino.spi.type.Timestamps.SECONDS_PER_DAY; import static io.trino.spi.type.Varchars.truncateToLength; import static java.lang.Math.floorDiv; import static java.lang.Math.floorMod; @@ -46,6 +49,9 @@ public final class TimestampCoercer .toFormatter() .withChronology(IsoChronology.INSTANCE); + // Before 1900, Java Time and Joda Time are not consistent with java.sql.Date and java.util.Calendar + private static final long START_OF_MODERN_ERA_SECONDS = java.time.LocalDate.of(1900, 1, 1).toEpochDay() * SECONDS_PER_DAY; + private TimestampCoercer() {} public static class LongTimestampToVarcharCoercer @@ -65,6 +71,9 @@ protected void applyCoercedValue(BlockBuilder blockBuilder, Block block, int pos long microsFraction = floorMod(timestamp.getEpochMicros(), MICROSECONDS_PER_SECOND); // Hive timestamp has nanoseconds precision, so no truncation here long nanosFraction = (microsFraction * NANOSECONDS_PER_MICROSECOND) + (timestamp.getPicosOfMicro() / PICOSECONDS_PER_NANOSECOND); + if (epochSecond < START_OF_MODERN_ERA_SECONDS) { + throw new TrinoException(HIVE_INVALID_TIMESTAMP_COERCION, "Coercion on historical dates is not supported"); + } toType.writeSlice( blockBuilder, diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestTimestampCoercer.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestTimestampCoercer.java index ed65927c80e2..12c3609fd666 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestTimestampCoercer.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestTimestampCoercer.java @@ -15,6 +15,7 @@ import io.airlift.slice.Slices; import io.trino.plugin.hive.HiveTimestampPrecision; +import io.trino.spi.TrinoException; import io.trino.spi.block.Block; import io.trino.spi.type.LongTimestamp; import io.trino.spi.type.SqlTimestamp; @@ -38,6 +39,7 @@ import static java.time.ZoneOffset.UTC; import static java.time.temporal.ChronoField.NANO_OF_SECOND; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestTimestampCoercer { @@ -86,11 +88,26 @@ public void testTimestampToSmallerVarchar() assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(29), "2023-04-11 05:16:12.345678876"); } + @Test + public void testHistoricalLongTimestampToVarchar() + { + LocalDateTime localDateTime = LocalDateTime.parse("1899-12-31T23:59:59.999999999"); + SqlTimestamp timestamp = SqlTimestamp.fromSeconds(TIMESTAMP_PICOS.getPrecision(), localDateTime.toEpochSecond(UTC), localDateTime.get(NANO_OF_SECOND)); + assertThatThrownBy(() -> assertLongTimestampToVarcharCoercions( + TIMESTAMP_PICOS, + new LongTimestamp(timestamp.getEpochMicros(), timestamp.getPicosOfMicros()), + createUnboundedVarcharType(), + "1899-12-31 23:59:59.999999999")) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Coercion on historical dates is not supported"); + } + @DataProvider public Object[][] timestampValuesProvider() { return new Object[][] { // before epoch + {"1900-01-01T00:00:00.000", "1900-01-01 00:00:00"}, {"1958-01-01T13:18:03.123", "1958-01-01 13:18:03.123"}, // after epoch {"2019-03-18T10:01:17.987", "2019-03-18 10:01:17.987"},