From 5d0dde210732f5bede30c6874b7f5e5f9ce4165a Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Sun, 28 Mar 2021 13:19:43 -0700 Subject: [PATCH 1/3] Fix support for Hive timestamp type --- .../apache/iceberg/hive/legacy/MergeHiveSchemaWithAvro.java | 4 +++- .../iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/MergeHiveSchemaWithAvro.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/MergeHiveSchemaWithAvro.java index 25bc3b4cf8..55018c7d77 100644 --- a/hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/MergeHiveSchemaWithAvro.java +++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/MergeHiveSchemaWithAvro.java @@ -236,7 +236,9 @@ public Schema hivePrimitiveToAvro(PrimitiveTypeInfo primitive) { return LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT)); case TIMESTAMP: - return LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)); + Schema schema = Schema.create(Schema.Type.LONG); + schema.addProp(AvroSchemaUtil.ADJUST_TO_UTC_PROP, false); + return LogicalTypes.timestampMillis().addToSchema(schema); case DECIMAL: DecimalTypeInfo dti = (DecimalTypeInfo) primitive; diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java index 5bc62beba7..5c86e97847 100644 --- a/hive-metastore/src/test/java/org/apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java +++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java @@ -278,14 +278,18 @@ public void shouldRecoverLogicalType() { optional("fc", Schema.Type.BYTES)); Schema merged = merge(hive, avro); + Schema expectedTimestampSchema = Schema.create(Schema.Type.LONG); + expectedTimestampSchema.addProp(AvroSchemaUtil.ADJUST_TO_UTC_PROP, false); Schema expected = struct("r1", optional("fa", LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT))), - optional("fb", LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG))), + optional("fb", LogicalTypes.timestampMillis().addToSchema(expectedTimestampSchema)), optional("fc", LogicalTypes.decimal(4, 2).addToSchema(Schema.create(Schema.Type.BYTES)))); assertSchema(expected, merged); Assert.assertEquals("date", AvroSchemaUtil.fromOption(merged.getField("fa").schema()).getLogicalType().getName()); + org.apache.iceberg.Schema iSchema = AvroSchemaUtil.toIceberg(merged); + System.out.println(iSchema); } // TODO: tests to retain schema props From 29efeebf9fac42a0fe4cc96b019cb0b19413aa0e Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Mon, 29 Mar 2021 14:47:55 -0700 Subject: [PATCH 2/3] Address pr comments --- .../apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java index 5c86e97847..8d30d487df 100644 --- a/hive-metastore/src/test/java/org/apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java +++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java @@ -288,8 +288,8 @@ public void shouldRecoverLogicalType() { assertSchema(expected, merged); Assert.assertEquals("date", AvroSchemaUtil.fromOption(merged.getField("fa").schema()).getLogicalType().getName()); + // This last line should not throw any exception. org.apache.iceberg.Schema iSchema = AvroSchemaUtil.toIceberg(merged); - System.out.println(iSchema); } // TODO: tests to retain schema props From b8118a6282263a4f6245d3fe8b98b152c94d754a Mon Sep 17 00:00:00 2001 From: Shardul Mahadik Date: Mon, 29 Mar 2021 14:52:05 -0700 Subject: [PATCH 3/3] Remove unnecessary assignment --- .../apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java index 8d30d487df..7ad76dae7a 100644 --- a/hive-metastore/src/test/java/org/apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java +++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/legacy/TestMergeHiveSchemaWithAvro.java @@ -289,7 +289,7 @@ public void shouldRecoverLogicalType() { Assert.assertEquals("date", AvroSchemaUtil.fromOption(merged.getField("fa").schema()).getLogicalType().getName()); // This last line should not throw any exception. - org.apache.iceberg.Schema iSchema = AvroSchemaUtil.toIceberg(merged); + AvroSchemaUtil.toIceberg(merged); } // TODO: tests to retain schema props