diff --git a/docs/content/iceberg/overview.md b/docs/content/iceberg/overview.md index 952bab40083e..35dc351905c1 100644 --- a/docs/content/iceberg/overview.md +++ b/docs/content/iceberg/overview.md @@ -96,8 +96,8 @@ Paimon Iceberg compatibility currently supports the following data types. | `BINARY` | `binary` | | `VARBINARY` | `binary` | | `DATE` | `date` | -| `TIMESTAMP` (precision 4-6) | `timestamp` | -| `TIMESTAMP_LTZ` (precision 4-6) | `timestamptz` | +| `TIMESTAMP` (precision 3-6) | `timestamp` | +| `TIMESTAMP_LTZ` (precision 3-6) | `timestamptz` | | `TIMESTAMP` (precision 7-9) | `timestamp_ns` | | `TIMESTAMP_LTZ` (precision 7-9) | `timestamptz_ns` | | `ARRAY` | `list` | @@ -106,6 +106,6 @@ Paimon Iceberg compatibility currently supports the following data types. {{< hint info >}} **Note on Timestamp Types:** -- `TIMESTAMP` and `TIMESTAMP_LTZ` types with precision from 4 to 6 are mapped to standard Iceberg timestamp types +- `TIMESTAMP` and `TIMESTAMP_LTZ` types with precision from 3 to 6 are mapped to standard Iceberg timestamp types - `TIMESTAMP` and `TIMESTAMP_LTZ` types with precision from 7 to 9 use nanosecond precision and require Iceberg v3 format -{{< /hint >}} \ No newline at end of file +{{< /hint >}} diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergConversions.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergConversions.java index a214ff1c8257..4998d0db14ec 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergConversions.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergConversions.java @@ -116,8 +116,8 @@ public static ByteBuffer toByteBuffer(DataType type, Object value) { private static ByteBuffer timestampToByteBuffer(Timestamp timestamp, int precision) { Preconditions.checkArgument( - precision > 3 && precision <= 6, - "Paimon Iceberg compatibility only support timestamp type with precision from 4 to 6."); + precision >= 3 && precision <= 6, + "Paimon Iceberg compatibility only support timestamp type with precision from 3 to 6."); return ByteBuffer.allocate(8) .order(ByteOrder.LITTLE_ENDIAN) .putLong(0, timestamp.toMicros()); @@ -157,8 +157,11 @@ public static Object toPaimonObject(DataType type, byte[] bytes) { long timestampLong = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).getLong(); Preconditions.checkArgument( - timestampPrecision > 3 && timestampPrecision <= 6, - "Paimon Iceberg compatibility only support timestamp type with precision from 4 to 6."); + timestampPrecision >= 3 && timestampPrecision <= 6, + "Paimon Iceberg compatibility only support timestamp type with precision from 3 to 6."); + if (timestampPrecision == 3) { + return Timestamp.fromEpochMillis(timestampLong); + } return Timestamp.fromMicros(timestampLong); default: throw new UnsupportedOperationException("Cannot deserialize type: " + type); diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java index 3714d59bfb24..0facdd47e456 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java @@ -174,14 +174,14 @@ private static Object toTypeObject(DataType dataType, int fieldId, int depth) { case TIMESTAMP_WITHOUT_TIME_ZONE: int timestampPrecision = ((TimestampType) dataType).getPrecision(); Preconditions.checkArgument( - timestampPrecision > 3 && timestampPrecision <= 9, - "Paimon Iceberg compatibility only support timestamp type with precision from 4 to 9."); + timestampPrecision >= 3 && timestampPrecision <= 9, + "Paimon Iceberg compatibility only support timestamp type with precision from 3 to 9."); return timestampPrecision >= 7 ? "timestamp_ns" : "timestamp"; case TIMESTAMP_WITH_LOCAL_TIME_ZONE: int timestampLtzPrecision = ((LocalZonedTimestampType) dataType).getPrecision(); Preconditions.checkArgument( - timestampLtzPrecision > 3 && timestampLtzPrecision <= 9, - "Paimon Iceberg compatibility only support timestamp type with precision from 4 to 9."); + timestampLtzPrecision >= 3 && timestampLtzPrecision <= 9, + "Paimon Iceberg compatibility only support timestamp type with precision from 3 to 9."); return timestampLtzPrecision >= 7 ? "timestamptz_ns" : "timestamptz"; case ARRAY: ArrayType arrayType = (ArrayType) dataType; diff --git a/paimon-core/src/test/java/org/apache/paimon/iceberg/manifest/IcebergConversionsTimestampTest.java b/paimon-core/src/test/java/org/apache/paimon/iceberg/manifest/IcebergConversionsTimestampTest.java new file mode 100644 index 000000000000..677b41d54db8 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/iceberg/manifest/IcebergConversionsTimestampTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.iceberg.manifest; + +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class IcebergConversionsTimestampTest { + + @ParameterizedTest + @MethodSource("provideTimestampConversionCases") + void testTimestampToByteBuffer(int precision, Timestamp input, long expectedMicros) { + ByteBuffer buffer = IcebergConversions.toByteBuffer(DataTypes.TIMESTAMP(precision), input); + assertThat(buffer.order()).isEqualTo(ByteOrder.LITTLE_ENDIAN); + assertThat(buffer.getLong(0)).isEqualTo(expectedMicros); + } + + private static Stream provideTimestampConversionCases() { + Timestamp tsMillis = + Timestamp.fromEpochMillis(1682164983524L); // 2023-04-22T13:03:03.524 (p=3) + Timestamp tsMicros = Timestamp.fromMicros(1683849603123456L); // 2023-05-12T00:00:03.123456 + + return Stream.of( + // For p=3..6 we encode microseconds per Iceberg spec + Arguments.of(3, tsMillis, 1682164983524000L), // micros from millis + Arguments.of(4, tsMillis, 1682164983524000L), + Arguments.of(5, tsMillis, 1682164983524000L), + Arguments.of(6, tsMillis, 1682164983524000L), + Arguments.of(6, tsMicros, 1683849603123456L)); // passthrough + } + + @ParameterizedTest + @MethodSource("provideInvalidPrecisions") + @DisplayName("Invalid timestamp precisions for ByteBuffer conversion") + void testTimestampToByteBufferInvalidPrecisions(int precision) { + Timestamp timestamp = Timestamp.fromEpochMillis(1682164983524L); + + assertThatThrownBy( + () -> + IcebergConversions.toByteBuffer( + DataTypes.TIMESTAMP(precision), timestamp)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining( + "Paimon Iceberg compatibility only support timestamp type with precision from 3 to 6."); + } + + private static Stream provideInvalidPrecisions() { + return Stream.of( + Arguments.of(0), + Arguments.of(1), + Arguments.of(2), + Arguments.of(7), + Arguments.of(8), + Arguments.of(9)); + } + + // ------------------------------------------------------------------------ + // toPaimonObject tests + // ------------------------------------------------------------------------ + + @ParameterizedTest + @MethodSource("provideTimestampToPaimonCases") + void testToPaimonObjectForTimestamp(int precision, long serializedMicros, String expectedTs) { + byte[] bytes = new byte[8]; + ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).putLong(serializedMicros); + + Timestamp actual = + (Timestamp) + IcebergConversions.toPaimonObject(DataTypes.TIMESTAMP(precision), bytes); + + assertThat(actual.toString()).isEqualTo(expectedTs); + } + + private static Stream provideTimestampToPaimonCases() { + return Stream.of( + // Provide binary in micros; p=3..6 should all parse as micros + Arguments.of(3, -1356022717123L, "1927-01-12T07:01:22.877"), + Arguments.of(3, 1713790983524L, "2024-04-22T13:03:03.524"), + Arguments.of(6, 1640690931207203L, "2021-12-28T11:28:51.207203")); + } + + @ParameterizedTest + @MethodSource("provideInvalidTimestampCases") + void testToPaimonObjectTimestampInvalid(int precision, long serializedMicros) { + byte[] bytes = new byte[8]; + ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).putLong(serializedMicros); + + assertThatThrownBy( + () -> + IcebergConversions.toPaimonObject( + DataTypes.TIMESTAMP(precision), bytes)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining( + "Paimon Iceberg compatibility only support timestamp type with precision from 3 to 6."); + } + + private static Stream provideInvalidTimestampCases() { + return Stream.of(Arguments.of(0, 1698686153L), Arguments.of(9, 1698686153123456789L)); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java b/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java index f96a0634598e..f4aff2a79b4f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java @@ -211,27 +211,27 @@ void testTimestampTypeConversions() { void testTimestampPrecisionValidation() { // Test invalid precision (<= 3) DataField invalidTimestampField = - new DataField(1, "timestamp", new TimestampType(false, 3)); + new DataField(1, "timestamp", new TimestampType(false, 2)); assertThatThrownBy(() -> new IcebergDataField(invalidTimestampField)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining( - "Paimon Iceberg compatibility only support timestamp type with precision from 4 to 9"); + "Paimon Iceberg compatibility only support timestamp type with precision from 3 to 9"); // Test invalid precision (<= 3) DataField invalidTimestampField2 = - new DataField(2, "timestamp", new TimestampType(false, 3)); + new DataField(2, "timestamp", new TimestampType(false, 2)); assertThatThrownBy(() -> new IcebergDataField(invalidTimestampField2)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining( - "Paimon Iceberg compatibility only support timestamp type with precision from 4 to 9"); + "Paimon Iceberg compatibility only support timestamp type with precision from 3 to 9"); // Test invalid local timezone timestamp precision (<= 3) DataField invalidTimestampLtzField = - new DataField(3, "timestamptz", new LocalZonedTimestampType(false, 3)); + new DataField(3, "timestamptz", new LocalZonedTimestampType(false, 2)); assertThatThrownBy(() -> new IcebergDataField(invalidTimestampLtzField)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining( - "Paimon Iceberg compatibility only support timestamp type with precision from 4 to 9"); + "Paimon Iceberg compatibility only support timestamp type with precision from 3 to 9"); // Test valid precision boundaries DataField validTimestamp4 = new DataField(4, "timestamp", new TimestampType(false, 4));