diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java index 518058700d315..9e4f74c7180d3 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java @@ -53,7 +53,6 @@ import java.io.IOException; import java.io.Serializable; -import java.time.Instant; import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Collections; @@ -523,8 +522,7 @@ public Option getEarliestCommitToRetain() { - commitsRetained); //15 instants total, 10 commits to retain, this gives 6th instant in the list } } else if (config.getCleanerPolicy() == HoodieCleaningPolicy.KEEP_LATEST_BY_HOURS) { - Instant instant = Instant.now(); - ZonedDateTime currentDateTime = ZonedDateTime.ofInstant(instant, hoodieTable.getMetaClient().getTableConfig().getTimelineTimezone().getZoneId()); + ZonedDateTime currentDateTime = ZonedDateTime.now(); String earliestTimeToRetain = HoodieActiveTimeline.formatDate(Date.from(currentDateTime.minusHours(hoursRetained).toInstant())); earliestCommitToRetain = Option.fromJavaOptional(commitTimeline.getInstantsAsStream().filter(i -> HoodieTimeline.compareTimestamps(i.getTimestamp(), HoodieTimeline.GREATER_THAN_OR_EQUALS, earliestTimeToRetain)).findFirst()); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstantTimeGenerator.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstantTimeGenerator.java index e839e73669e90..18ec9e7ae5ada 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstantTimeGenerator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstantTimeGenerator.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.table.timeline; import org.apache.hudi.common.model.HoodieTimelineTimeZone; +import org.apache.hudi.common.table.HoodieTableConfig; import java.text.ParseException; import java.time.LocalDateTime; @@ -44,10 +45,10 @@ public class HoodieInstantTimeGenerator { public static final int MILLIS_INSTANT_TIMESTAMP_FORMAT_LENGTH = MILLIS_INSTANT_TIMESTAMP_FORMAT.length(); // Formatter to generate Instant timestamps // Unfortunately millisecond format is not parsable as is https://bugs.openjdk.java.net/browse/JDK-8031085. hence have to do appendValue() - private static DateTimeFormatter MILLIS_INSTANT_TIME_FORMATTER = new DateTimeFormatterBuilder().appendPattern(SECS_INSTANT_TIMESTAMP_FORMAT) + private static final DateTimeFormatter MILLIS_INSTANT_TIME_FORMATTER = new DateTimeFormatterBuilder().appendPattern(SECS_INSTANT_TIMESTAMP_FORMAT) .appendValue(ChronoField.MILLI_OF_SECOND, 3).toFormatter(); private static final String MILLIS_GRANULARITY_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; - private static DateTimeFormatter MILLIS_GRANULARITY_DATE_FORMATTER = DateTimeFormatter.ofPattern(MILLIS_GRANULARITY_DATE_FORMAT); + private static final DateTimeFormatter MILLIS_GRANULARITY_DATE_FORMATTER = DateTimeFormatter.ofPattern(MILLIS_GRANULARITY_DATE_FORMAT); // The last Instant timestamp generated private static AtomicReference lastInstantTime = new AtomicReference<>(String.valueOf(Integer.MIN_VALUE)); @@ -61,7 +62,7 @@ public class HoodieInstantTimeGenerator { /** * Returns next instant time that adds N milliseconds to the current time. - * Ensures each instant time is atleast 1 second apart since we create instant times at second granularity + * Ensures each instant time is at least 1 second apart since we create instant times at second granularity * * @param milliseconds Milliseconds to add to current time while generating the new instant time */ @@ -94,7 +95,7 @@ public static Date parseDateFromInstantTime(String timestamp) throws ParseExcept } LocalDateTime dt = LocalDateTime.parse(timestampInMillis, MILLIS_INSTANT_TIME_FORMATTER); - return Date.from(dt.atZone(ZoneId.systemDefault()).toInstant()); + return Date.from(dt.atZone(getZoneId()).toInstant()); } catch (DateTimeParseException e) { throw new ParseException(e.getMessage(), e.getErrorIndex()); } @@ -129,7 +130,7 @@ public static String getInstantForDateString(String dateString) { } private static TemporalAccessor convertDateToTemporalAccessor(Date d) { - return d.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + return d.toInstant().atZone(getZoneId()).toLocalDateTime(); } public static void setCommitTimeZone(HoodieTimelineTimeZone commitTimeZone) { @@ -144,4 +145,11 @@ public static boolean isValidInstantTime(String instantTime) { return false; } } + + private static ZoneId getZoneId() { + HoodieTimelineTimeZone timelineTimezone = new HoodieTableConfig().getTimelineTimezone(); + return timelineTimezone.equals(HoodieTimelineTimeZone.LOCAL) + ? ZoneId.systemDefault() + : ZoneId.of(commitTimeZone.getTimeZone().toUpperCase()); + } } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java index b7311fcc37c19..72deb7fbb6424 100755 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java @@ -20,6 +20,7 @@ import org.apache.hudi.common.fs.HoodieWrapperFileSystem; import org.apache.hudi.common.fs.NoOpConsistencyGuard; +import org.apache.hudi.common.model.HoodieTimelineTimeZone; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant.State; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; @@ -37,6 +38,8 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.text.ParseException; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -648,6 +651,22 @@ public void testInvalidInstantDateParsing() throws ParseException { HoodieActiveTimeline.parseDateFromInstantTimeSafely(testInstant).get().getTime()); } + @Test + public void testFormatDateWithCommitTimeZone() throws ParseException { + String testInstant = "20230508120101"; + Date date = HoodieActiveTimeline.parseDateFromInstantTime(testInstant); + + HoodieInstantTimeGenerator.setCommitTimeZone(HoodieTimelineTimeZone.LOCAL); + LocalDateTime localDateTime = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + String defaultStr = HoodieInstantTimeGenerator.getInstantFromTemporalAccessor(localDateTime); + assertEquals(defaultStr, HoodieActiveTimeline.formatDate(date)); + + HoodieInstantTimeGenerator.setCommitTimeZone(HoodieTimelineTimeZone.UTC); + LocalDateTime utcDateTime = date.toInstant().atZone(ZoneId.of("UTC")).toLocalDateTime(); + String utcStr = HoodieInstantTimeGenerator.getInstantFromTemporalAccessor(utcDateTime); + assertEquals(utcStr, HoodieActiveTimeline.formatDate(date)); + } + /** * Returns an exhaustive list of all possible HoodieInstant. * @return list of HoodieInstant