diff --git a/common/src/java/org/apache/hadoop/hive/common/type/TimestampNano.java b/common/src/java/org/apache/hadoop/hive/common/type/TimestampNano.java new file mode 100644 index 000000000000..e31c43761405 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/type/TimestampNano.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.type; + +public class TimestampNano extends Timestamp { + + public TimestampNano(Timestamp ts) { + super(ts); + } +} diff --git a/common/src/java/org/apache/hadoop/hive/common/type/TimestampNanoTZ.java b/common/src/java/org/apache/hadoop/hive/common/type/TimestampNanoTZ.java new file mode 100644 index 000000000000..9441940c03d6 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/type/TimestampNanoTZ.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.type; + +import java.time.ZonedDateTime; + +public class TimestampNanoTZ extends TimestampTZ { + public TimestampNanoTZ(ZonedDateTime zonedDateTime) { + super(zonedDateTime); + } +} diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java index 330f67e32bb2..e10e6c7626ea 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java @@ -136,6 +136,10 @@ Type convertType(TypeInfo typeInfo, String defaultValue) { case DECIMAL: DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo; return Types.DecimalType.of(decimalTypeInfo.precision(), decimalTypeInfo.scale()); + case TIMESTAMP_NS: + return Types.TimestampNanoType.withoutZone(); + case TIMESTAMPTZ_NS: + return Types.TimestampNanoType.withZone(); case INTERVAL_YEAR_MONTH: case INTERVAL_DAY_TIME: default: diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java index 8277546b5255..b48728c5b232 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java @@ -399,6 +399,12 @@ public static String convertToTypeString(Type type) { return String.format("map<%s,%s>", convert(mapType.keyType()), convert(mapType.valueType())); case VARIANT: return "variant"; + case TIMESTAMP_NANO: + Types.TimestampNanoType nanoType = (Types.TimestampNanoType) type; + if (nanoType.shouldAdjustToUTC()) { + return "nanosecond timestamp with local time zone"; + } + return "nanosecond timestamp"; default: throw new UnsupportedOperationException(type + " is not supported"); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergNanosecondTimestampObjectInspectorHive3.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergNanosecondTimestampObjectInspectorHive3.java new file mode 100644 index 000000000000..10e950880fd3 --- /dev/null +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergNanosecondTimestampObjectInspectorHive3.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive.serde.objectinspector; + +import org.apache.hadoop.hive.common.type.Timestamp; +import org.apache.hadoop.hive.common.type.TimestampNano; +import org.apache.hadoop.hive.serde2.io.TimestampNanoWritable; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +public class IcebergNanosecondTimestampObjectInspectorHive3 extends IcebergTimestampObjectInspectorHive3 { + + private static final IcebergNanosecondTimestampObjectInspectorHive3 INSTANCE = + new IcebergNanosecondTimestampObjectInspectorHive3(); + + public static IcebergNanosecondTimestampObjectInspectorHive3 get() { + return INSTANCE; + } + + private IcebergNanosecondTimestampObjectInspectorHive3() { + super(TypeInfoFactory.timestampNanoTypeInfo); + } + + @Override + public TimestampNano getPrimitiveJavaObject(Object o) { + Timestamp timestamp = super.getPrimitiveJavaObject(o); + if (timestamp == null) { + return null; + } + return new TimestampNano(timestamp); + } + + @Override + public TimestampNanoWritable getPrimitiveWritableObject(Object o) { + TimestampNano ts = getPrimitiveJavaObject(o); + return ts == null ? null : new TimestampNanoWritable(ts); + } +} diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergNanosecondTimestampWithZoneObjectInspectorHive3.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergNanosecondTimestampWithZoneObjectInspectorHive3.java new file mode 100644 index 000000000000..7ad9922b0d22 --- /dev/null +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergNanosecondTimestampWithZoneObjectInspectorHive3.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive.serde.objectinspector; + +import org.apache.hadoop.hive.common.type.TimestampNanoTZ; +import org.apache.hadoop.hive.common.type.TimestampTZ; +import org.apache.hadoop.hive.serde2.io.TimestampNanoTZWritable; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +public class IcebergNanosecondTimestampWithZoneObjectInspectorHive3 + extends IcebergTimestampWithZoneObjectInspectorHive3 { + + private static final IcebergNanosecondTimestampWithZoneObjectInspectorHive3 INSTANCE = + new IcebergNanosecondTimestampWithZoneObjectInspectorHive3(); + + public static IcebergNanosecondTimestampWithZoneObjectInspectorHive3 get() { + return INSTANCE; + } + + private IcebergNanosecondTimestampWithZoneObjectInspectorHive3() { + super(TypeInfoFactory.timestampNanosTZTypeInfo); + } + + @Override + public TimestampNanoTZ getPrimitiveJavaObject(Object o) { + TimestampTZ ttz = super.getPrimitiveJavaObject(o); + if (ttz == null) { + return null; + } + return new TimestampNanoTZ(ttz.getZonedDateTime()); + } + + @Override + public TimestampNanoTZWritable getPrimitiveWritableObject(Object o) { + TimestampNanoTZ tsTz = getPrimitiveJavaObject(o); + return tsTz == null ? null : new TimestampNanoTZWritable(tsTz); + } +} diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java index 0951e30128a0..0084aa40c96a 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java @@ -135,6 +135,11 @@ public ObjectInspector primitive(Type.PrimitiveType primitiveType) { return adjustToUTC ? TIMESTAMP_INSPECTOR_WITH_TZ : TIMESTAMP_INSPECTOR; case TIME: return IcebergTimeObjectInspector.get(); + case TIMESTAMP_NANO: + boolean adjust = ((Types.TimestampNanoType) primitiveType).shouldAdjustToUTC(); + return adjust ? + IcebergNanosecondTimestampWithZoneObjectInspectorHive3.get() : + IcebergNanosecondTimestampObjectInspectorHive3.get(); default: throw new IllegalArgumentException(primitiveType.typeId() + " type is not supported"); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspectorHive3.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspectorHive3.java index f2e5c12dab8a..22058df819f0 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspectorHive3.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspectorHive3.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -42,6 +43,10 @@ private IcebergTimestampObjectInspectorHive3() { super(TypeInfoFactory.timestampTypeInfo); } + protected IcebergTimestampObjectInspectorHive3(PrimitiveTypeInfo typeInfo) { + super(typeInfo); + } + @Override public LocalDateTime convert(Object o) { if (o == null) { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampWithZoneObjectInspectorHive3.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampWithZoneObjectInspectorHive3.java index 864f3db4d9d3..e98adace5a9c 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampWithZoneObjectInspectorHive3.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampWithZoneObjectInspectorHive3.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable; import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -43,6 +44,10 @@ private IcebergTimestampWithZoneObjectInspectorHive3() { super(TypeInfoFactory.timestampLocalTZTypeInfo); } + protected IcebergTimestampWithZoneObjectInspectorHive3(PrimitiveTypeInfo typeInfo) { + super(typeInfo); + } + @Override public OffsetDateTime convert(Object o) { if (o == null) { diff --git a/iceberg/iceberg-handler/src/test/queries/positive/timestamp_ns.q b/iceberg/iceberg-handler/src/test/queries/positive/timestamp_ns.q new file mode 100644 index 000000000000..4a294ae7e99c --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/timestamp_ns.q @@ -0,0 +1,46 @@ +-- Mask random uuid +--! qt:replace:/(\s+'uuid'=')\S+('\s*)/$1#Masked#$2/ +--! qt:replace:/(\s+uuid\s+)\S+/$1#Masked#/ +-- Mask random snapshot id +--! qt:replace:/('current-snapshot-id'=')\d+/$1#SnapshotId#/ +-- Mask current-snapshot-timestamp-ms +--! qt:replace:/('current-snapshot-timestamp-ms'=')\d+/$1#Masked#/ +-- Mask iceberg version +--! qt:replace:/("iceberg-version":")(\w+\s\w+\s\d+\.\d+\.\d+\s\(\w+\s\w+\))/$1#Masked#/ +-- Mask added-files-size +--! qt:replace:/(\S\"added-files-size":")(\d+)(")/$1#Masked#$3/ +-- Mask total-files-size +--! qt:replace:/(\S\"total-files-size":")(\d+)(")/$1#Masked#$3/ + +CREATE TABLE t ( + ts_us timestamp, + ts_ns nanosecond timestamp, + ts_tz_us timestamp with local time zone, + ts_tz_ns nanosecond timestamp with local time zone +) +STORED BY ICEBERG +TBLPROPERTIES ('format-version'='3'); + +SHOW CREATE TABLE t; +DESCRIBE t; + +INSERT INTO t VALUES ( + '2025-12-18 10:15:30.123456789', + '2025-12-18 10:15:30.123456789', + '2025-12-18 10:15:30.123456789', + '2025-12-18 10:15:30.123456789' +); + +SELECT ts_ns FROM t ORDER BY ts_ns; +SELECT ts_tz_ns FROM t ORDER BY ts_tz_ns; +SELECT CAST(ts_ns AS STRING) FROM t; +SELECT CAST(ts_tz_ns AS STRING) FROM t; + +SELECT * FROM t; + +CREATE TABLE tgt STORED BY ICEBERG TBLPROPERTIES ('format-version'='3') AS SELECT * FROM t; + +SHOW CREATE TABLE tgt; +DESCRIBE tgt; + +SELECT * FROM tgt; diff --git a/iceberg/iceberg-handler/src/test/results/positive/timestamp_ns.q.out b/iceberg/iceberg-handler/src/test/results/positive/timestamp_ns.q.out new file mode 100644 index 000000000000..ca2a3afc2571 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/timestamp_ns.q.out @@ -0,0 +1,202 @@ +PREHOOK: query: CREATE TABLE t ( + ts_us timestamp, + ts_ns nanosecond timestamp, + ts_tz_us timestamp with local time zone, + ts_tz_ns nanosecond timestamp with local time zone +) +STORED BY ICEBERG +TBLPROPERTIES ('format-version'='3') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: CREATE TABLE t ( + ts_us timestamp, + ts_ns nanosecond timestamp, + ts_tz_us timestamp with local time zone, + ts_tz_ns nanosecond timestamp with local time zone +) +STORED BY ICEBERG +TBLPROPERTIES ('format-version'='3') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: SHOW CREATE TABLE t +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@t +POSTHOOK: query: SHOW CREATE TABLE t +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@t +CREATE EXTERNAL TABLE `t`( + `ts_us` timestamp, + `ts_ns` nanosecond timestamp, + `ts_tz_us` timestamp with local time zone, + `ts_tz_ns` nanosecond timestamp with local time zone) +ROW FORMAT SERDE + 'org.apache.iceberg.mr.hive.HiveIcebergSerDe' +STORED BY + 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler' + +LOCATION + 'hdfs://### HDFS PATH ###' +TBLPROPERTIES ( + 'bucketing_version'='2', + 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"ts_us","required":false,"type":"timestamp"},{"id":2,"name":"ts_ns","required":false,"type":"timestamp_ns"},{"id":3,"name":"ts_tz_us","required":false,"type":"timestamptz"},{"id":4,"name":"ts_tz_ns","required":false,"type":"timestamptz_ns"}]}', + 'format-version'='3', + 'iceberg.orc.files.only'='false', + 'metadata_location'='hdfs://### HDFS PATH ###', + 'parquet.compression'='zstd', + 'serialization.format'='1', + 'snapshot-count'='0', + 'table_type'='ICEBERG', +#### A masked pattern was here #### + 'uuid'='#Masked#', + 'write.delete.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.metadata.delete-after-commit.enabled'='true', + 'write.update.mode'='merge-on-read') +PREHOOK: query: DESCRIBE t +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@t +POSTHOOK: query: DESCRIBE t +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@t +ts_us timestamp +ts_ns nanosecond timestamp +ts_tz_us timestamp with local time zone +ts_tz_ns nanosecond timestamp with local time zone +PREHOOK: query: INSERT INTO t VALUES ( + '2025-12-18 10:15:30.123456789', + '2025-12-18 10:15:30.123456789', + '2025-12-18 10:15:30.123456789', + '2025-12-18 10:15:30.123456789' +) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t +POSTHOOK: query: INSERT INTO t VALUES ( + '2025-12-18 10:15:30.123456789', + '2025-12-18 10:15:30.123456789', + '2025-12-18 10:15:30.123456789', + '2025-12-18 10:15:30.123456789' +) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t +PREHOOK: query: SELECT ts_ns FROM t ORDER BY ts_ns +PREHOOK: type: QUERY +PREHOOK: Input: default@t +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT ts_ns FROM t ORDER BY ts_ns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +POSTHOOK: Output: hdfs://### HDFS PATH ### +2025-12-18 10:15:30.123456789 +PREHOOK: query: SELECT ts_tz_ns FROM t ORDER BY ts_tz_ns +PREHOOK: type: QUERY +PREHOOK: Input: default@t +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT ts_tz_ns FROM t ORDER BY ts_tz_ns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +POSTHOOK: Output: hdfs://### HDFS PATH ### +2025-12-18 10:15:30.123456789 US/Pacific +PREHOOK: query: SELECT CAST(ts_ns AS STRING) FROM t +PREHOOK: type: QUERY +PREHOOK: Input: default@t +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT CAST(ts_ns AS STRING) FROM t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +POSTHOOK: Output: hdfs://### HDFS PATH ### +2025-12-18 10:15:30.123456789 +PREHOOK: query: SELECT CAST(ts_tz_ns AS STRING) FROM t +PREHOOK: type: QUERY +PREHOOK: Input: default@t +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT CAST(ts_tz_ns AS STRING) FROM t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +POSTHOOK: Output: hdfs://### HDFS PATH ### +2025-12-18 10:15:30.123456789 US/Pacific +PREHOOK: query: SELECT * FROM t +PREHOOK: type: QUERY +PREHOOK: Input: default@t +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +POSTHOOK: Output: hdfs://### HDFS PATH ### +2025-12-18 10:15:30.123456 2025-12-18 10:15:30.123456789 2025-12-18 10:15:30.123456 US/Pacific 2025-12-18 10:15:30.123456789 US/Pacific +PREHOOK: query: CREATE TABLE tgt STORED BY ICEBERG TBLPROPERTIES ('format-version'='3') AS SELECT * FROM t +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@t +PREHOOK: Output: database:default +PREHOOK: Output: default@tgt +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: CREATE TABLE tgt STORED BY ICEBERG TBLPROPERTIES ('format-version'='3') AS SELECT * FROM t +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@t +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tgt +POSTHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: Lineage: tgt.ts_ns SIMPLE [(t)t.FieldSchema(name:ts_ns, type:nanosecond timestamp, comment:null), ] +POSTHOOK: Lineage: tgt.ts_tz_ns SIMPLE [(t)t.FieldSchema(name:ts_tz_ns, type:nanosecond timestamp with local time zone, comment:null), ] +POSTHOOK: Lineage: tgt.ts_tz_us SIMPLE [(t)t.FieldSchema(name:ts_tz_us, type:timestamp with local time zone, comment:null), ] +POSTHOOK: Lineage: tgt.ts_us SIMPLE [(t)t.FieldSchema(name:ts_us, type:timestamp, comment:null), ] +PREHOOK: query: SHOW CREATE TABLE tgt +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@tgt +POSTHOOK: query: SHOW CREATE TABLE tgt +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@tgt +CREATE EXTERNAL TABLE `tgt`( + `ts_us` timestamp, + `ts_ns` nanosecond timestamp, + `ts_tz_us` timestamp with local time zone, + `ts_tz_ns` nanosecond timestamp with local time zone) +ROW FORMAT SERDE + 'org.apache.iceberg.mr.hive.HiveIcebergSerDe' +STORED BY + 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler' + +LOCATION + 'hdfs://### HDFS PATH ###' +TBLPROPERTIES ( + 'bucketing_version'='2', + 'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"ts_us","required":false,"type":"timestamp"},{"id":2,"name":"ts_ns","required":false,"type":"timestamp_ns"},{"id":3,"name":"ts_tz_us","required":false,"type":"timestamptz"},{"id":4,"name":"ts_tz_ns","required":false,"type":"timestamptz_ns"}]}', + 'current-snapshot-id'='#SnapshotId#', + 'current-snapshot-summary'='{"added-data-files":"1","added-records":"1","added-files-size":"#Masked#","changed-partition-count":"1","total-records":"1","total-files-size":"#Masked#","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"#Masked#"}', + 'current-snapshot-timestamp-ms'='#Masked#', + 'format-version'='3', + 'iceberg.orc.files.only'='false', + 'metadata_location'='hdfs://### HDFS PATH ###', + 'parquet.compression'='zstd', + 'previous_metadata_location'='hdfs://### HDFS PATH ###', + 'serialization.format'='1', + 'snapshot-count'='1', + 'table_type'='ICEBERG', +#### A masked pattern was here #### + 'uuid'='#Masked#', + 'write.delete.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.metadata.delete-after-commit.enabled'='true', + 'write.update.mode'='merge-on-read') +PREHOOK: query: DESCRIBE tgt +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tgt +POSTHOOK: query: DESCRIBE tgt +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tgt +ts_us timestamp +ts_ns nanosecond timestamp +ts_tz_us timestamp with local time zone +ts_tz_ns nanosecond timestamp with local time zone +PREHOOK: query: SELECT * FROM tgt +PREHOOK: type: QUERY +PREHOOK: Input: default@tgt +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM tgt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tgt +POSTHOOK: Output: hdfs://### HDFS PATH ### +2025-12-18 10:15:30.123456 2025-12-18 10:15:30.123456789 2025-12-18 10:15:30.123456 US/Pacific 2025-12-18 10:15:30.123456789 US/Pacific diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g index cb3404587d86..214683e7e2ad 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g @@ -116,6 +116,9 @@ KW_DATE: 'DATE'; KW_DATETIME: 'DATETIME'; KW_TIMESTAMP: 'TIMESTAMP'; KW_TIMESTAMPLOCALTZ: 'TIMESTAMPLOCALTZ'; +KW_TIMESTAMP_NS: 'TIMESTAMP_NS'; +KW_TIMESTAMPTZ_NS: 'TIMESTAMPTZ_NS'; +KW_NANOSECOND: 'NANOSECOND'; KW_TIME: 'TIME'; KW_ZONE: 'ZONE'; KW_INTERVAL: 'INTERVAL'; diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 1fa3a011328e..1dc85570f20c 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -137,6 +137,10 @@ TOK_TIMESTAMP; TOK_TIMESTAMPLITERAL; TOK_TIMESTAMPLOCALTZ; TOK_TIMESTAMPLOCALTZLITERAL; +TOK_TIMESTAMP_NS; +TOK_TIMESTAMPTZ_NS; +TOK_TIMESTAMP_NSLITERAL; +TOK_TIMESTAMPTZ_NSLITERAL; TOK_INTERVAL_YEAR_MONTH; TOK_INTERVAL_YEAR_MONTH_LITERAL; TOK_INTERVAL_DAY_TIME; @@ -2445,6 +2449,8 @@ primitiveType | KW_DECIMAL (LPAREN prec=Number (COMMA scale=Number)? RPAREN)? -> ^(TOK_DECIMAL $prec? $scale?) | KW_VARCHAR LPAREN length=Number RPAREN -> ^(TOK_VARCHAR $length) | KW_CHAR LPAREN length=Number RPAREN -> ^(TOK_CHAR $length) + | KW_NANOSECOND KW_TIMESTAMP -> TOK_TIMESTAMP_NS + | KW_NANOSECOND KW_TIMESTAMP KW_WITH KW_LOCAL KW_TIME KW_ZONE -> TOK_TIMESTAMPTZ_NS ; listType diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index ac9053cf284e..8e245a0dcd38 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -453,6 +453,8 @@ constant | dateLiteral | timestampLiteral | timestampLocalTZLiteral + | timestampNsLiteral + | timestampTzNsLiteral | StringLiteral | stringLiteralSequence | charSetStringLiteral @@ -521,6 +523,16 @@ timestampLocalTZLiteral } ; +timestampNsLiteral + : KW_NANOSECOND KW_TIMESTAMP StringLiteral + -> ^(TOK_TIMESTAMP_NSLITERAL StringLiteral) + ; + +timestampTzNsLiteral + : KW_NANOSECOND KW_TIMESTAMP KW_WITH KW_LOCAL KW_TIME KW_ZONE StringLiteral + -> ^(TOK_TIMESTAMPTZ_NSLITERAL StringLiteral) + ; + intervalValue : StringLiteral | Number diff --git a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestReservedWords.java b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestReservedWords.java index 50491173384e..e0005cb9f4b6 100644 --- a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestReservedWords.java +++ b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestReservedWords.java @@ -174,7 +174,10 @@ public static Collection data() { "WHERE", "WINDOW", "WITH", - "VARIANT" + "VARIANT", + "TIMESTAMP_NS", + "TIMESTAMPTZ_NS", + "NANOSECOND" ); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index eda1360293fe..4a73cd98d122 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -525,6 +525,8 @@ public final class FunctionRegistry { system.registerGenericUDF(serdeConstants.DATE_TYPE_NAME, GenericUDFToDate.class); system.registerGenericUDF(serdeConstants.TIMESTAMP_TYPE_NAME, GenericUDFTimestamp.class); system.registerGenericUDF(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME, GenericUDFToTimestampLocalTZ.class); + system.registerGenericUDF(serdeConstants.TIMESTAMP_NS_TYPE_NAME, GenericUDFTimestamp.class); + system.registerGenericUDF(serdeConstants.TIMESTAMPTZ_NS_TYPE_NAME, GenericUDFToTimestampLocalTZ.class); system.registerGenericUDF(serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME, GenericUDFToIntervalYearMonth.class); system.registerGenericUDF(serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME, GenericUDFToIntervalDayTime.class); system.registerGenericUDF(serdeConstants.BINARY_TYPE_NAME, GenericUDFToBinary.class); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 000eb8a6d919..485e1822cb7c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -88,6 +88,8 @@ import com.google.common.base.Preconditions; import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.hadoop.hive.serde.serdeConstants.TIMESTAMPTZ_NS_TYPE_NAME; +import static org.apache.hadoop.hive.serde.serdeConstants.TIMESTAMP_NS_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.VARIANT_TYPE_NAME; /** @@ -1157,12 +1159,13 @@ public static void validateColumns(List columns, List for (FieldSchema col: columns) { String colName = normalize(col.getName()); if (colNames.contains(colName)) { - throw new HiveException("Duplicate column name " + colName - + " in the table definition."); + throw new HiveException("Duplicate column name " + colName + " in the table definition."); } - if (!icebergTable && VARIANT_TYPE_NAME.equalsIgnoreCase(col.getType())) { - throw new HiveException( - "Column name " + colName + " cannot be of type 'variant' as it is not supported in non-Iceberg tables."); + if (!icebergTable && (VARIANT_TYPE_NAME.equalsIgnoreCase(col.getType()) || + TIMESTAMP_NS_TYPE_NAME.equalsIgnoreCase(col.getType()) || + TIMESTAMPTZ_NS_TYPE_NAME.equalsIgnoreCase(col.getType()))) { + throw new HiveException("Column name " + colName + " cannot be of type '" + col.getType() + + "' as it is not supported in non-Iceberg tables" + "."); } colNames.add(colName); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java index d191ecef829f..2b6f1fbca195 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java @@ -210,9 +210,11 @@ public static RelDataType convert(PrimitiveTypeInfo type, boolean nullable, RelD convertedType = dtFactory.createSqlType(SqlTypeName.DATE); break; case TIMESTAMP: + case TIMESTAMP_NS: convertedType = dtFactory.createSqlType(SqlTypeName.TIMESTAMP); break; case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: convertedType = dtFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE); break; case INTERVAL_YEAR_MONTH: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 393006bd8f06..ac1f3d19e0c2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1046,6 +1046,8 @@ public static String getTypeStringFromAST(ASTNode typeNode) TOKEN_TO_TYPE.put(HiveParser.TOK_INTERVAL_DAY_TIME, serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME); TOKEN_TO_TYPE.put(HiveParser.TOK_DECIMAL, serdeConstants.DECIMAL_TYPE_NAME); TOKEN_TO_TYPE.put(HiveParser.TOK_VARIANT, serdeConstants.VARIANT_TYPE_NAME); + TOKEN_TO_TYPE.put(HiveParser.TOK_TIMESTAMP_NS, serdeConstants.TIMESTAMP_NS_TYPE_NAME); + TOKEN_TO_TYPE.put(HiveParser.TOK_TIMESTAMPTZ_NS, serdeConstants.TIMESTAMPTZ_NS_TYPE_NAME); } private static String getTypeName(ASTNode node) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index f08808b01415..829f0efd9195 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -290,6 +290,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java index 1eb1d17297f7..8a0c32065d4b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java @@ -142,6 +142,8 @@ public class TypeCheckProcFactory { CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_MAP, "toMap"); CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_LIST, "toArray"); CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_STRUCT, "toStruct"); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_TIMESTAMP_NS, serdeConstants.TIMESTAMP_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_TIMESTAMPTZ_NS, serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME); WINDOWING_TOKENS = new HashSet(); WINDOWING_TOKENS.add(HiveParser.KW_OVER); @@ -518,6 +520,22 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } return exprFactory.createTimestampLocalTimeZoneConstantExpr(timeString, conf.getLocalTimeZone()); } + if (expr.getType() == HiveParser.TOK_TIMESTAMP_NSLITERAL) { + T ts = exprFactory.createTimestampConstantExpr(timeString); + return exprFactory.setTypeInfo(ts, TypeInfoFactory.timestampNanoTypeInfo); + } + if (expr.getType() == HiveParser.TOK_TIMESTAMPTZ_NSLITERAL) { + HiveConf conf; + try { + conf = Hive.get().getConf(); + } catch (HiveException e) { + throw new SemanticException(e); + } + + T ts = exprFactory.createTimestampLocalTimeZoneConstantExpr(timeString, conf.getLocalTimeZone()); + return exprFactory.setTypeInfo(ts, TypeInfoFactory.timestampNanosTZTypeInfo); + } + throw new IllegalArgumentException("Invalid time literal type " + expr.getType()); } catch (Exception err) { throw new SemanticException( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index be62d94019ed..a37aa5a55b41 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -390,6 +390,7 @@ public static ColumnStatsType getColumnStatsType(PrimitiveTypeInfo typeInfo) case INT: case LONG: case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: return LONG; case FLOAT: case DOUBLE: @@ -399,6 +400,7 @@ public static ColumnStatsType getColumnStatsType(PrimitiveTypeInfo typeInfo) case DATE: return DATE; case TIMESTAMP: + case TIMESTAMP_NS: return TIMESTAMP; case STRING: case CHAR: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java index 55043413839a..cf80c7b82c01 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java @@ -105,6 +105,9 @@ public void configure(MapredContext context) { @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { + if (tsConvertors[0] instanceof ObjectInspectorConverters.IdentityConverter) { + return arguments[0].get(); + } PrimitiveObjectInspectorConverter.TimestampConverter ts = (PrimitiveObjectInspectorConverter.TimestampConverter) tsConvertors[0]; ts.setIntToTimestampInSeconds(intToTimestampInSeconds); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java index 5701fc40581e..3366b0249e37 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java @@ -96,6 +96,8 @@ void testPrimitiveSizeEstimations() throws Exception { exclusions.add(serdeConstants.STRUCT_TYPE_NAME); exclusions.add(serdeConstants.UNION_TYPE_NAME); exclusions.add(serdeConstants.VARIANT_TYPE_NAME); + exclusions.add(serdeConstants.TIMESTAMP_NS_TYPE_NAME); + exclusions.add(serdeConstants.TIMESTAMPTZ_NS_TYPE_NAME); Field[] serdeFields = serdeConstants.class.getFields(); for (Field field : serdeFields) { if (!Modifier.isStatic(field.getModifiers())) { diff --git a/ql/src/test/queries/clientnegative/timestamp_ns_add_column.q b/ql/src/test/queries/clientnegative/timestamp_ns_add_column.q new file mode 100644 index 000000000000..17d853d60c85 --- /dev/null +++ b/ql/src/test/queries/clientnegative/timestamp_ns_add_column.q @@ -0,0 +1,2 @@ +create table emp(id int); +alter table emp add columns (t nanosecond timestamp); \ No newline at end of file diff --git a/ql/src/test/queries/clientnegative/timestamp_ns_non_iceberg.q b/ql/src/test/queries/clientnegative/timestamp_ns_non_iceberg.q new file mode 100644 index 000000000000..9b51d87e618e --- /dev/null +++ b/ql/src/test/queries/clientnegative/timestamp_ns_non_iceberg.q @@ -0,0 +1 @@ +CREATE EXTERNAL TABLE variant_test_basic (id INT, t nanosecond timestamp with local time zone); \ No newline at end of file diff --git a/ql/src/test/results/clientnegative/timestamp_ns_add_column.q.out b/ql/src/test/results/clientnegative/timestamp_ns_add_column.q.out new file mode 100644 index 000000000000..adfa2a3c4af9 --- /dev/null +++ b/ql/src/test/results/clientnegative/timestamp_ns_add_column.q.out @@ -0,0 +1,13 @@ +PREHOOK: query: create table emp(id int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@emp +POSTHOOK: query: create table emp(id int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@emp +PREHOOK: query: alter table emp add columns (t nanosecond timestamp) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@emp +PREHOOK: Output: default@emp +FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. Column name t cannot be of type 'nanosecond timestamp' as it is not supported in non-Iceberg tables. diff --git a/ql/src/test/results/clientnegative/timestamp_ns_non_iceberg.q.out b/ql/src/test/results/clientnegative/timestamp_ns_non_iceberg.q.out new file mode 100644 index 000000000000..26bdefbf4913 --- /dev/null +++ b/ql/src/test/results/clientnegative/timestamp_ns_non_iceberg.q.out @@ -0,0 +1,5 @@ +PREHOOK: query: CREATE EXTERNAL TABLE variant_test_basic (id INT, t nanosecond timestamp with local time zone) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@variant_test_basic +FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. org.apache.hadoop.hive.ql.metadata.HiveException: Column name t cannot be of type 'nanosecond timestamp with local time zone' as it is not supported in non-Iceberg tables. diff --git a/ql/src/test/results/clientpositive/llap/show_functions.q.out b/ql/src/test/results/clientpositive/llap/show_functions.q.out index 605e953d6349..861ad19053e1 100644 --- a/ql/src/test/results/clientpositive/llap/show_functions.q.out +++ b/ql/src/test/results/clientpositive/llap/show_functions.q.out @@ -283,6 +283,8 @@ month months_between murmur_hash named_struct +nanosecond timestamp +nanosecond timestamp with local time zone ndv_compute_bit_vector negative next_day @@ -607,6 +609,7 @@ lcase like locate minute +nanosecond timestamp with local time zone negative ntile parse_url_tuple @@ -925,6 +928,8 @@ month months_between murmur_hash named_struct +nanosecond timestamp +nanosecond timestamp with local time zone ndv_compute_bit_vector negative next_day diff --git a/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java b/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java index 20e607a67bdb..6879c6fd2a70 100644 --- a/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java +++ b/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java @@ -95,6 +95,9 @@ public class serdeConstants { public static final java.lang.String VARIANT_TYPE_NAME = "variant"; + public static final String TIMESTAMP_NS_TYPE_NAME = "nanosecond timestamp"; + public static final String TIMESTAMPTZ_NS_TYPE_NAME = "nanosecond timestamp with local time zone"; + public static final java.lang.String LIST_COLUMNS = "columns"; public static final java.lang.String LIST_COLUMN_TYPES = "columns.types"; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java index f962a29edebc..13bbbe6bd79a 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java @@ -352,6 +352,7 @@ static Object deserialize(InputByteBuffer buffer, TypeInfo type, } case TIMESTAMP: + case TIMESTAMP_NS: TimestampWritableV2 t = (reuse == null ? new TimestampWritableV2() : (TimestampWritableV2) reuse); byte[] bytes = new byte[TimestampWritableV2.BINARY_SORTABLE_LENGTH]; @@ -362,6 +363,7 @@ static Object deserialize(InputByteBuffer buffer, TypeInfo type, t.setBinarySortable(bytes, 0); return t; case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: TimestampLocalTZWritable tstz = (reuse == null ? new TimestampLocalTZWritable() : (TimestampLocalTZWritable) reuse); byte[] data = new byte[TimestampLocalTZWritable.BINARY_SORTABLE_LENGTH]; @@ -733,13 +735,15 @@ static void serialize(ByteStream.Output buffer, Object o, ObjectInspector oi, serializeInt(buffer, v, invert); return; } - case TIMESTAMP: { + case TIMESTAMP: + case TIMESTAMP_NS:{ TimestampObjectInspector toi = (TimestampObjectInspector) poi; TimestampWritableV2 t = toi.getPrimitiveWritableObject(o); serializeTimestampWritable(buffer, t, invert); return; } - case TIMESTAMPLOCALTZ: { + case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS:{ TimestampLocalTZObjectInspector toi = (TimestampLocalTZObjectInspector) poi; TimestampLocalTZWritable t = toi.getPrimitiveWritableObject(o); serializeTimestampTZWritable(buffer, t, invert); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampNanoTZWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampNanoTZWritable.java new file mode 100644 index 000000000000..b0004982fb3e --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampNanoTZWritable.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.io; + +import org.apache.hadoop.hive.common.type.TimestampNanoTZ; + +public class TimestampNanoTZWritable extends TimestampLocalTZWritable { + public TimestampNanoTZWritable(TimestampNanoTZ tsTz) { + super(tsTz); + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampNanoWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampNanoWritable.java new file mode 100644 index 000000000000..100abe357b81 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampNanoWritable.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.io; + +import org.apache.hadoop.hive.common.type.TimestampNano; + +public class TimestampNanoWritable extends TimestampWritableV2 { + public TimestampNanoWritable(TimestampNano ts) { + super(ts); + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java index 65a76ac5ef27..08dbbbf18688 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java @@ -303,12 +303,14 @@ public static void writePrimitiveUTF8(OutputStream out, Object o, ((DateObjectInspector) oi).getPrimitiveWritableObject(o)); break; } - case TIMESTAMP: { + case TIMESTAMP: + case TIMESTAMP_NS: { LazyTimestamp.writeUTF8(out, ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o)); break; } - case TIMESTAMPLOCALTZ: { + case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: { LazyTimestampLocalTZ.writeUTF8(out, ((TimestampLocalTZObjectInspector) oi). getPrimitiveWritableObject(o)); break; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java index 7e69ffc8c8a4..c4ba1a131af0 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java @@ -118,6 +118,8 @@ private LazyPrimitiveObjectInspectorFactory() { LAZY_INTERVAL_DAY_TIME_OBJECT_INSPECTOR); cachedPrimitiveLazyObjectInspectors.put(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME), LAZY_BINARY_OBJECT_INSPECTOR); + cachedPrimitiveLazyObjectInspectors.put(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_NS_TYPE_NAME), + LAZY_TIMESTAMP_OBJECT_INSPECTOR); } public static AbstractPrimitiveLazyObjectInspector getLazyObjectInspector( diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java index 07e1408f104c..c79e195fc19e 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java @@ -133,10 +133,12 @@ private static Converter getConverter(PrimitiveObjectInspector inputOI, inputOI, (SettableDateObjectInspector) outputOI); case TIMESTAMP: + case TIMESTAMP_NS: return new PrimitiveObjectInspectorConverter.TimestampConverter( inputOI, (SettableTimestampObjectInspector) outputOI); case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: return new PrimitiveObjectInspectorConverter.TimestampLocalTZConverter( inputOI, (SettableTimestampLocalTZObjectInspector) outputOI); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java index 5f148b4da137..f78c6e470829 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java @@ -36,7 +36,8 @@ public interface PrimitiveObjectInspector extends ObjectInspector { enum PrimitiveCategory { VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, DATE, TIMESTAMP, TIMESTAMPLOCALTZ, BINARY, DECIMAL, VARCHAR, CHAR, - INTERVAL_YEAR_MONTH, INTERVAL_DAY_TIME, UNKNOWN + INTERVAL_YEAR_MONTH, INTERVAL_DAY_TIME, UNKNOWN, TIMESTAMP_NS, + TIMESTAMPTZ_NS } PrimitiveTypeInfo getTypeInfo(); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java index 47719c856429..5d1b97035e0e 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java @@ -19,6 +19,7 @@ import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; public class JavaTimestampObjectInspector @@ -29,6 +30,10 @@ protected JavaTimestampObjectInspector() { super(TypeInfoFactory.timestampTypeInfo); } + public JavaTimestampObjectInspector(PrimitiveTypeInfo typeInfo) { + super(typeInfo); + } + public TimestampWritableV2 getPrimitiveWritableObject(Object o) { return o == null ? null : new TimestampWritableV2((Timestamp) o); } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java index 84c027d51c9a..c6810ce5ac29 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.common.type.Timestamp; +import org.apache.hadoop.hive.common.type.TimestampNanoTZ; import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.serde2.ByteStream; import org.apache.hadoop.hive.serde2.lazy.LazyInteger; @@ -489,10 +490,12 @@ public Text convert(Object input) { t.set(((DateObjectInspector) inputOI).getPrimitiveWritableObject(input).toString()); return t; case TIMESTAMP: + case TIMESTAMP_NS: t.set(((TimestampObjectInspector) inputOI) .getPrimitiveWritableObject(input).toString()); return t; case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: t.set(((TimestampLocalTZObjectInspector) inputOI).getPrimitiveWritableObject(input).toString()); return t; case INTERVAL_YEAR_MONTH: diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java index 0b31d9a9a266..1dd2c0112a1b 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java @@ -99,6 +99,10 @@ public final class PrimitiveObjectInspectorFactory { new WritableBinaryObjectInspector(); public static final WritableHiveDecimalObjectInspector writableHiveDecimalObjectInspector = new WritableHiveDecimalObjectInspector(TypeInfoFactory.decimalTypeInfo); + public static final WritableTimestampObjectInspector writableTimestampNanoObjectInspector = + new WritableTimestampObjectInspector(TypeInfoFactory.timestampNanoTypeInfo); + public static final WritableTimestampLocalTZObjectInspector writableTimestampNanoTZObjectInspector = + new WritableTimestampLocalTZObjectInspector(TypeInfoFactory.timestampNanosTZTypeInfo); // Map from PrimitiveTypeInfo to AbstractPrimitiveWritableObjectInspector. private static ConcurrentHashMap @@ -139,6 +143,8 @@ public final class PrimitiveObjectInspectorFactory { cachedPrimitiveWritableInspectorCache.put(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME), writableBinaryObjectInspector); cachedPrimitiveWritableInspectorCache.put(TypeInfoFactory.decimalTypeInfo, writableHiveDecimalObjectInspector); + cachedPrimitiveWritableInspectorCache.put(TypeInfoFactory.timestampNanoTypeInfo, writableTimestampNanoObjectInspector); + cachedPrimitiveWritableInspectorCache.put(TypeInfoFactory.timestampNanosTZTypeInfo, writableTimestampNanoTZObjectInspector); } private static Map @@ -165,6 +171,8 @@ public final class PrimitiveObjectInspectorFactory { primitiveCategoryToWritableOI.put(PrimitiveCategory.INTERVAL_DAY_TIME, writableHiveIntervalDayTimeObjectInspector); primitiveCategoryToWritableOI.put(PrimitiveCategory.BINARY, writableBinaryObjectInspector); primitiveCategoryToWritableOI.put(PrimitiveCategory.DECIMAL, writableHiveDecimalObjectInspector); + primitiveCategoryToWritableOI.put(PrimitiveCategory.TIMESTAMP_NS, writableTimestampNanoObjectInspector); + primitiveCategoryToWritableOI.put(PrimitiveCategory.TIMESTAMPTZ_NS, writableTimestampNanoTZObjectInspector); } public static final JavaBooleanObjectInspector javaBooleanObjectInspector = @@ -240,6 +248,10 @@ public final class PrimitiveObjectInspectorFactory { cachedPrimitiveJavaInspectorCache.put(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME), javaByteArrayObjectInspector); cachedPrimitiveJavaInspectorCache.put(TypeInfoFactory.decimalTypeInfo, javaHiveDecimalObjectInspector); + cachedPrimitiveJavaInspectorCache.put(TypeInfoFactory.timestampNanoTypeInfo, + new JavaTimestampObjectInspector(TypeInfoFactory.timestampNanoTypeInfo)); + cachedPrimitiveJavaInspectorCache.put(TypeInfoFactory.timestampNanosTZTypeInfo, + new JavaTimestampLocalTZObjectInspector(TypeInfoFactory.timestampNanosTZTypeInfo)); } private static Map primitiveCategoryToJavaOI = @@ -263,6 +275,10 @@ public final class PrimitiveObjectInspectorFactory { primitiveCategoryToJavaOI.put(PrimitiveCategory.INTERVAL_DAY_TIME, javaHiveIntervalDayTimeObjectInspector); primitiveCategoryToJavaOI.put(PrimitiveCategory.BINARY, javaByteArrayObjectInspector); primitiveCategoryToJavaOI.put(PrimitiveCategory.DECIMAL, javaHiveDecimalObjectInspector); + primitiveCategoryToJavaOI.put(PrimitiveCategory.TIMESTAMP_NS, + new JavaTimestampObjectInspector(TypeInfoFactory.timestampNanoTypeInfo)); + primitiveCategoryToJavaOI.put(PrimitiveCategory.TIMESTAMPTZ_NS, + new JavaTimestampLocalTZObjectInspector(TypeInfoFactory.timestampNanosTZTypeInfo)); } /** @@ -302,6 +318,7 @@ public static AbstractPrimitiveWritableObjectInspector getPrimitiveWritableObjec result = new WritableHiveVarcharObjectInspector((VarcharTypeInfo) typeInfo); break; case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: result = new WritableTimestampLocalTZObjectInspector((TimestampLocalTZTypeInfo) typeInfo); break; case DECIMAL: @@ -410,6 +427,7 @@ public static AbstractPrimitiveJavaObjectInspector getPrimitiveJavaObjectInspect result = new JavaHiveVarcharObjectInspector((VarcharTypeInfo)typeInfo); break; case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: result = new JavaTimestampLocalTZObjectInspector((TimestampLocalTZTypeInfo)typeInfo); break; case DECIMAL: diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java index c61b4c3c581c..63f0e5cd3066 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java @@ -37,6 +37,8 @@ import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.common.type.Timestamp; +import org.apache.hadoop.hive.common.type.TimestampNano; +import org.apache.hadoop.hive.common.type.TimestampNanoTZ; import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.common.type.TimestampTZUtil; import org.apache.hadoop.hive.common.type.TimestampUtils; @@ -52,6 +54,8 @@ import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable; +import org.apache.hadoop.hive.serde2.io.TimestampNanoTZWritable; +import org.apache.hadoop.hive.serde2.io.TimestampNanoWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.lazy.LazyInteger; import org.apache.hadoop.hive.serde2.lazy.LazyLong; @@ -254,6 +258,13 @@ static void registerType(PrimitiveTypeEntry t) { PrimitiveCategory.CHAR, serdeConstants.CHAR_TYPE_NAME, null, HiveChar.class, HiveCharWritable.class); + public static final PrimitiveTypeEntry timestampnsTypeEntry = + new PrimitiveTypeEntry(PrimitiveCategory.TIMESTAMP_NS, serdeConstants.TIMESTAMP_NS_TYPE_NAME, null, + TimestampNano.class, TimestampNanoWritable.class); + public static final PrimitiveTypeEntry timestampnsTZTypeEntry = + new PrimitiveTypeEntry(PrimitiveCategory.TIMESTAMPTZ_NS, serdeConstants.TIMESTAMPTZ_NS_TYPE_NAME, null, + TimestampNanoTZ.class, TimestampNanoTZWritable.class); + // The following is a complex type for special handling public static final PrimitiveTypeEntry unknownTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.UNKNOWN, "unknown", null, Object.class, null); @@ -277,6 +288,8 @@ static void registerType(PrimitiveTypeEntry t) { registerType(intervalYearMonthTypeEntry); registerType(intervalDayTimeTypeEntry); registerType(decimalTypeEntry); + registerType(timestampnsTypeEntry); + registerType(timestampnsTZTypeEntry); registerType(unknownTypeEntry); } @@ -448,11 +461,13 @@ public static boolean comparePrimitiveObjects(Object o1, return ((DateObjectInspector) oi1).getPrimitiveWritableObject(o1) .equals(((DateObjectInspector) oi2).getPrimitiveWritableObject(o2)); } - case TIMESTAMP: { + case TIMESTAMP: + case TIMESTAMP_NS: { return ((TimestampObjectInspector) oi1).getPrimitiveWritableObject(o1) .equals(((TimestampObjectInspector) oi2).getPrimitiveWritableObject(o2)); } - case TIMESTAMPLOCALTZ: { + case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: { return ((TimestampLocalTZObjectInspector) oi1).getPrimitiveWritableObject(o1).equals( ((TimestampLocalTZObjectInspector) oi2).getPrimitiveWritableObject(o2)); } @@ -542,6 +557,7 @@ public static boolean getBoolean(Object o, PrimitiveObjectInspector oi) { } break; case TIMESTAMP: + case TIMESTAMP_NS: result = (((TimestampObjectInspector) oi) .getPrimitiveWritableObject(o).getSeconds() != 0); break; @@ -734,6 +750,7 @@ public static int getInt(Object o, PrimitiveObjectInspector oi) { break; } case TIMESTAMP: + case TIMESTAMP_NS: result = (int) (((TimestampObjectInspector) oi) .getPrimitiveWritableObject(o).getSeconds()); break; @@ -804,10 +821,12 @@ public static long getLong(Object o, PrimitiveObjectInspector oi) { break; } case TIMESTAMP: + case TIMESTAMP_NS: result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) .getSeconds(); break; case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: result = ((TimestampLocalTZObjectInspector) oi).getPrimitiveWritableObject(o) .getSeconds(); break; @@ -871,6 +890,7 @@ public static double getDouble(Object o, PrimitiveObjectInspector oi) { result = Double.parseDouble(getString(o, oi)); break; case TIMESTAMP: + case TIMESTAMP_NS: result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).getDouble(); break; case DECIMAL: @@ -957,9 +977,11 @@ public static String getString(Object o, PrimitiveObjectInspector oi) { result = ((DateObjectInspector) oi).getPrimitiveWritableObject(o).toString(); break; case TIMESTAMP: + case TIMESTAMP_NS: result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).toString(); break; case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: result = ((TimestampLocalTZObjectInspector) oi).getPrimitiveWritableObject(o).toString(); break; case INTERVAL_YEAR_MONTH: @@ -1100,6 +1122,7 @@ public static HiveDecimal getHiveDecimal(Object o, PrimitiveObjectInspector oi) result = HiveDecimal.create(getString(o, oi)); break; case TIMESTAMP: + case TIMESTAMP_NS: Double ts = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) .getDouble(); result = HiveDecimal.create(ts.toString()); @@ -1153,10 +1176,12 @@ public static Date getDate(Object o, PrimitiveObjectInspector oi) { result = ((DateObjectInspector) oi).getPrimitiveWritableObject(o).get(); break; case TIMESTAMP: + case TIMESTAMP_NS: result = DateWritableV2.timeToDate( ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).getSeconds()); break; case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: String tstz = oi.getPrimitiveWritableObject(o).toString(); int divSpace = tstz.indexOf(' '); if (divSpace == -1) { @@ -1231,9 +1256,11 @@ public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector inputOI, ((DateObjectInspector) inputOI).getPrimitiveWritableObject(o).get().toEpochMilli()); break; case TIMESTAMP: + case TIMESTAMP_NS: result = ((TimestampObjectInspector) inputOI).getPrimitiveWritableObject(o).getTimestamp(); break; case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: String tstz = inputOI.getPrimitiveWritableObject(o).toString(); int index = tstz.indexOf(" "); index = tstz.indexOf(" ", index + 1); @@ -1318,11 +1345,13 @@ public static TimestampTZ getTimestampLocalTZ(Object o, PrimitiveObjectInspector Date date = ((DateObjectInspector) oi).getPrimitiveWritableObject(o).get(); return TimestampTZUtil.convert(date, timeZone); } - case TIMESTAMP: { + case TIMESTAMP: + case TIMESTAMP_NS: { Timestamp ts = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).getTimestamp(); return TimestampTZUtil.convert(ts, timeZone); } - case TIMESTAMPLOCALTZ: { + case TIMESTAMPLOCALTZ: + case TIMESTAMPTZ_NS: { return ((TimestampLocalTZObjectInspector) oi).getPrimitiveWritableObject(o).getTimestampTZ(); } default: @@ -1440,6 +1469,8 @@ public static PrimitiveGrouping getPrimitiveGrouping(PrimitiveCategory primitive case DATE: case TIMESTAMP: case TIMESTAMPLOCALTZ: + case TIMESTAMP_NS: + case TIMESTAMPTZ_NS: return PrimitiveGrouping.DATE_GROUP; case INTERVAL_YEAR_MONTH: case INTERVAL_DAY_TIME: diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java index e0ab191b73cc..5acc359c5cbf 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java @@ -19,6 +19,7 @@ import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; public class WritableTimestampObjectInspector extends @@ -29,6 +30,10 @@ public WritableTimestampObjectInspector() { super(TypeInfoFactory.timestampTypeInfo); } + public WritableTimestampObjectInspector(PrimitiveTypeInfo typeInfo) { + super(typeInfo); + } + @Override public TimestampWritableV2 getPrimitiveWritableObject(Object o) { return o == null ? null : (TimestampWritableV2) o; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/thrift/Type.java b/serde/src/java/org/apache/hadoop/hive/serde2/thrift/Type.java index 9df1b8dc6132..52021ee49895 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/thrift/Type.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/thrift/Type.java @@ -75,6 +75,12 @@ public enum Type { TIMESTAMPLOCALTZ_TYPE(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME.toUpperCase(), java.sql.Types.OTHER, TTypeId.TIMESTAMPLOCALTZ_TYPE), + TIMESTAMPNS_TYPE(serdeConstants.TIMESTAMP_NS_TYPE_NAME.toUpperCase(), + java.sql.Types.TIMESTAMP, + TTypeId.TIMESTAMP_TYPE), + TIMESTAMPTZ_NS_TYPE(serdeConstants.TIMESTAMPTZ_NS_TYPE_NAME.toUpperCase(), + java.sql.Types.OTHER, + TTypeId.TIMESTAMPLOCALTZ_TYPE), INTERVAL_YEAR_MONTH_TYPE("INTERVAL_YEAR_MONTH", java.sql.Types.OTHER, TTypeId.INTERVAL_YEAR_MONTH_TYPE), @@ -230,9 +236,15 @@ public static Type getType(TypeInfo typeInfo) { case TIMESTAMP: { return Type.TIMESTAMP_TYPE; } + case TIMESTAMP_NS: { + return Type.TIMESTAMPNS_TYPE; + } case TIMESTAMPLOCALTZ: { return Type.TIMESTAMPLOCALTZ_TYPE; } + case TIMESTAMPTZ_NS: { + return Type.TIMESTAMPTZ_NS_TYPE; + } case INTERVAL_YEAR_MONTH: { return Type.INTERVAL_YEAR_MONTH_TYPE; } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java index e1f9a2699a8e..07af959ac1b9 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java @@ -38,9 +38,9 @@ public TimestampLocalTZTypeInfo(String timeZoneStr) { this.timeZone = TimestampTZUtil.parseTimeZone(timeZoneStr); } - @Override - public String getTypeName() { - return serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME; + public TimestampLocalTZTypeInfo(String typeName, String timeZoneStr) { + super(typeName); + this.timeZone = TimestampTZUtil.parseTimeZone(timeZoneStr); } @Override @@ -78,11 +78,15 @@ public String toString() { @Override public String getQualifiedName() { - return getQualifiedName(null); + return getQualifiedName(typeName, null); } public static String getQualifiedName(ZoneId timeZone) { - StringBuilder sb = new StringBuilder(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME); + return getQualifiedName(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME, timeZone); + } + + public static String getQualifiedName(String typeName, ZoneId timeZone) { + StringBuilder sb = new StringBuilder(typeName); if (timeZone != null) { sb.append("('"); sb.append(timeZone); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java index 977bbd0277f9..c77b8c67387a 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java @@ -59,6 +59,7 @@ private TypeInfoFactory() { public static final PrimitiveTypeInfo intervalYearMonthTypeInfo = new PrimitiveTypeInfo(serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME); public static final PrimitiveTypeInfo intervalDayTimeTypeInfo = new PrimitiveTypeInfo(serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME); public static final PrimitiveTypeInfo binaryTypeInfo = new PrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME); + public static final PrimitiveTypeInfo timestampNanoTypeInfo = new PrimitiveTypeInfo(serdeConstants.TIMESTAMP_NS_TYPE_NAME); /** * A DecimalTypeInfo instance that has max precision and max scale. @@ -72,6 +73,9 @@ private TypeInfoFactory() { public static final TimestampLocalTZTypeInfo timestampLocalTZTypeInfo = new TimestampLocalTZTypeInfo( ZoneId.systemDefault().getId()); + public static final TimestampLocalTZTypeInfo timestampNanosTZTypeInfo = + new TimestampLocalTZTypeInfo(serdeConstants.TIMESTAMPTZ_NS_TYPE_NAME, ZoneId.systemDefault().getId()); + public static final PrimitiveTypeInfo unknownTypeInfo = new PrimitiveTypeInfo("unknown"); // Map from type name (such as int or varchar(40) to the corresponding PrimitiveTypeInfo @@ -98,6 +102,8 @@ private TypeInfoFactory() { cachedPrimitiveTypeInfo.put(serdeConstants.BINARY_TYPE_NAME, binaryTypeInfo); cachedPrimitiveTypeInfo.put(decimalTypeInfo.getQualifiedName(), decimalTypeInfo); cachedPrimitiveTypeInfo.put("unknown", unknownTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.TIMESTAMP_NS_TYPE_NAME, timestampNanoTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.TIMESTAMPTZ_NS_TYPE_NAME, timestampNanosTZTypeInfo); } /** diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java index 9db912182425..5d570b697dc0 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java @@ -84,6 +84,12 @@ public class ColumnType { public static final String VARIANT_TYPE_NAME = "variant"; + public static final String TIMESTAMP_NS = "nanosecond timestamp"; + + public static final String TIMESTAMPTZ_NS = "nanosecond timestamp with local time zone"; + + public static final String NANOSECOND = "nanosecond"; + public static final String LIST_COLUMNS = "columns"; public static final String LIST_COLUMN_TYPES = "columns.types"; @@ -109,7 +115,9 @@ public class ColumnType { INTERVAL_DAY_TIME_TYPE_NAME, DECIMAL_TYPE_NAME, BINARY_TYPE_NAME, - TIMESTAMPTZ_TYPE_NAME); + TIMESTAMPTZ_TYPE_NAME, + TIMESTAMPTZ_NS, + TIMESTAMP_NS); public static final Set StringTypes = StringUtils.asSet( STRING_TYPE_NAME, @@ -200,6 +208,7 @@ public class ColumnType { STRUCT_TYPE_NAME, UNION_TYPE_NAME, VARIANT_TYPE_NAME, + NANOSECOND, LIST_COLUMNS, LIST_COLUMN_TYPES, COLUMN_NAME_DELIMITER