diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/v2/InstantDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/v2/InstantDTO.java new file mode 100644 index 0000000000000..18515ceba1598 --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/v2/InstantDTO.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.table.timeline.dto.v2; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.InstantGenerator; + +/** + * The data transfer object of instant. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class InstantDTO { + + @JsonProperty("action") + String action; + @JsonProperty("requestTs") + String requestedTime; + @JsonProperty("completionTs") + String completionTime; + @JsonProperty("state") + String state; + + public static InstantDTO fromInstant(HoodieInstant instant) { + if (null == instant) { + return null; + } + + InstantDTO dto = new InstantDTO(); + dto.action = instant.getAction(); + dto.requestedTime = instant.requestedTime(); + dto.completionTime = instant.getCompletionTime(); + dto.state = instant.getState().toString(); + return dto; + } + + public static HoodieInstant toInstant(InstantDTO dto, InstantGenerator factory) { + if (null == dto) { + return null; + } + + return factory.createNewInstant(HoodieInstant.State.valueOf(dto.state), dto.action, + dto.requestedTime, dto.completionTime); + } +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/v2/TimelineDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/v2/TimelineDTO.java new file mode 100644 index 0000000000000..66d73224a1db9 --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/v2/TimelineDTO.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.table.timeline.dto.v2; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.table.timeline.InstantGenerator; +import org.apache.hudi.common.table.timeline.TimelineFactory; + +/** + * The data transfer object of timeline. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class TimelineDTO { + + @JsonProperty("instants") + List instants; + + public static TimelineDTO fromTimeline(HoodieTimeline timeline) { + TimelineDTO dto = new TimelineDTO(); + dto.instants = timeline.getInstantsAsStream().map(InstantDTO::fromInstant).collect(Collectors.toList()); + return dto; + } + + public static HoodieTimeline toTimeline(TimelineDTO dto, HoodieTableMetaClient metaClient) { + InstantGenerator instantGenerator = metaClient.getInstantGenerator(); + TimelineFactory factory = metaClient.getTimelineLayout().getTimelineFactory(); + // TODO: For Now, we will assume, only active-timeline will be transferred. + return factory.createDefaultTimeline(dto.instants.stream().map(d -> InstantDTO.toInstant(d, instantGenerator)), + metaClient.getActiveTimeline()); + } +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java index b7e42b69759eb..dc78d2e3e053d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java @@ -68,6 +68,7 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView, private static final String SCHEME = "http"; private static final String BASE_URL = "/v1/hoodie/view"; + private static final String BASE_URL_V2 = "/v2/hoodie/view"; public static final String LATEST_PARTITION_SLICES_URL = String.format("%s/%s", BASE_URL, "slices/partition/latest/"); public static final String LATEST_PARTITION_SLICES_INFLIGHT_URL = String.format("%s/%s", BASE_URL, "slices/partition/latest/inflight/"); public static final String LATEST_PARTITION_SLICES_STATELESS_URL = String.format("%s/%s", BASE_URL, "slices/partition/latest/stateless/"); @@ -103,7 +104,10 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView, public static final String LAST_INSTANT_URL = String.format("%s/%s", BASE_URL, "timeline/instant/last"); public static final String LAST_INSTANTS_URL = String.format("%s/%s", BASE_URL, "timeline/instants/last"); + public static final String INSTANT_DETAILS_URL = String.format("%s/%s", BASE_URL_V2, "timeline/instant"); + public static final String TIMELINE_URL = String.format("%s/%s", BASE_URL, "timeline/instants/all"); + public static final String TIMELINE_V2_URL = String.format("%s/%s", BASE_URL_V2, "timeline/instants/all"); // POST Requests public static final String REFRESH_TABLE_URL = String.format("%s/%s", BASE_URL, "refresh/"); @@ -114,6 +118,8 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView, public static final String PARTITIONS_PARAM = "partitions"; public static final String BASEPATH_PARAM = "basepath"; public static final String INSTANT_PARAM = "instant"; + public static final String INSTANT_ACTION_PARAM = "instantaction"; + public static final String INSTANT_STATE_PARAM = "instantstate"; public static final String MAX_INSTANT_PARAM = "maxinstant"; public static final String MIN_INSTANT_PARAM = "mininstant"; public static final String INSTANTS_PARAM = "instants"; diff --git a/hudi-timeline-service/pom.xml b/hudi-timeline-service/pom.xml index cf99434de1ab7..97cf72556f7a8 100644 --- a/hudi-timeline-service/pom.xml +++ b/hudi-timeline-service/pom.xml @@ -122,6 +122,12 @@ ${javalin.version} + + org.thymeleaf + thymeleaf + ${thymeleaf.version} + + com.beust jcommander diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/RequestHandler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/RequestHandler.java index 626c3836198fe..afc4783ee5511 100644 --- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/RequestHandler.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/RequestHandler.java @@ -173,6 +173,18 @@ private static String getMinInstantParam(Context ctx) { return ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.MIN_INSTANT_PARAM, String.class).getOrDefault(""); } + private static String getInstantParam(Context ctx) { + return ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.INSTANT_PARAM, String.class).getOrDefault(""); + } + + private static String getInstantActionParam(Context ctx) { + return ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.INSTANT_ACTION_PARAM, String.class).getOrDefault(""); + } + + private static String getInstantStateParam(Context ctx) { + return ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.INSTANT_STATE_PARAM, String.class).getOrDefault(""); + } + private static String getMarkerDirParam(Context ctx) { return ctx.queryParamAsClass(MarkerOperation.MARKER_DIR_PATH_PARAM, String.class).getOrDefault(""); } @@ -246,6 +258,19 @@ private void registerTimelineAPI() { TimelineDTO dto = instantHandler.getTimeline(getBasePathParam(ctx)); writeValueAsString(ctx, dto); }, false)); + + app.get(RemoteHoodieTableFileSystemView.TIMELINE_V2_URL, new ViewHandler(ctx -> { + metricsRegistry.add("TIMELINE_V2", 1); + org.apache.hudi.common.table.timeline.dto.v2.TimelineDTO dto = instantHandler.getTimelineV2(getBasePathParam(ctx)); + writeValueAsString(ctx, dto); + }, false)); + + app.get(RemoteHoodieTableFileSystemView.INSTANT_DETAILS_URL, new ViewHandler(ctx -> { + metricsRegistry.add("INSTANT_DETAILS", 1); + Object instantDetailsObj = instantHandler.getInstantDetails(getBasePathParam(ctx), + getInstantParam(ctx), getInstantActionParam(ctx), getInstantStateParam(ctx)); + writeValueAsString(ctx, instantDetailsObj); + }, false)); } /** diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java index e9300fc0dd970..fd75854b59d37 100644 --- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java @@ -32,6 +32,7 @@ import com.beust.jcommander.Parameter; import io.javalin.Javalin; import io.javalin.core.util.JavalinBindException; +import org.apache.hudi.timeline.service.ui.UiHandler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler; @@ -57,6 +58,7 @@ public class TimelineService { private transient Javalin app = null; private transient FileSystemViewManager fsViewsManager; private transient RequestHandler requestHandler; + private transient UiHandler uiHandler; public int getServerPort() { return serverPort; @@ -387,8 +389,20 @@ private void createApp() { requestHandler = new RequestHandler( app, storageConf, timelineServerConf, context, fsViewsManager); + uiHandler = new UiHandler(app); app.get("/", ctx -> ctx.result("Hello Hudi")); requestHandler.register(); + uiHandler.register(); + // API endpoint for returning timeline events dynamically + app.get("/api/timeline", ctx -> { + // Return some dynamic event data + String timelineData = "[" + + "{\"id\": 1, \"content\": \"Event 1\", \"start\": \"2025-04-01\"}," + + "{\"id\": 2, \"content\": \"Event 2\", \"start\": \"2025-04-02\"}," + + "{\"id\": 3, \"content\": \"Event 3\", \"start\": \"2025-04-03\"}" + + "]"; + ctx.result(timelineData).contentType("application/json"); + }); } public void run() throws IOException { diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/TimelineHandler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/TimelineHandler.java index da29f859c2167..8f81ff4ee177a 100644 --- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/TimelineHandler.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/TimelineHandler.java @@ -18,8 +18,12 @@ package org.apache.hudi.timeline.service.handlers; +import java.io.IOException; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.dto.InstantDTO; import org.apache.hudi.common.table.timeline.dto.TimelineDTO; +import org.apache.hudi.common.table.timeline.versioning.v1.InstantComparatorV1; import org.apache.hudi.common.table.view.FileSystemViewManager; import org.apache.hudi.storage.StorageConfiguration; import org.apache.hudi.timeline.service.TimelineService; @@ -46,4 +50,30 @@ public List getLastInstant(String basePath) { public TimelineDTO getTimeline(String basePath) { return TimelineDTO.fromTimeline(viewManager.getFileSystemView(basePath).getTimeline()); } + + public org.apache.hudi.common.table.timeline.dto.v2.TimelineDTO getTimelineV2(String basePath) { + return org.apache.hudi.common.table.timeline.dto.v2.TimelineDTO.fromTimeline(viewManager.getFileSystemView(basePath).getTimeline()); + } + + public Object getInstantDetails(String basePath, String requestedTime, String action, String state) + throws IOException { + HoodieTimeline hoodieTimeline = viewManager.getFileSystemView(basePath).getTimeline(); + try { + HoodieInstant requestedInstant = new HoodieInstant( + HoodieInstant.State.valueOf(state), action, requestedTime, + InstantComparatorV1.REQUESTED_TIME_BASED_COMPARATOR); + + switch (requestedInstant.getAction()) { + case HoodieTimeline.COMMIT_ACTION: + case HoodieTimeline.DELTA_COMMIT_ACTION: + return hoodieTimeline.readCommitMetadata(requestedInstant); + default: + // TODO: implement other actions + return null; + } + } catch (Exception e) { + // Input parameters might be invalid + return null; + } + } } diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/ui/TimelineView.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/ui/TimelineView.java new file mode 100644 index 0000000000000..8e7c49314c5ed --- /dev/null +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/ui/TimelineView.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.timeline.service.ui; + +import io.javalin.http.Handler; +import java.util.HashMap; +import java.util.Map; + +public class TimelineView { + + public static Handler serveTimelineView = ctx -> { + Map model = new HashMap<>(); + model.put("title", "Vis Timeline Example/Demo"); + ctx.render("/templates/timeline.html", model); + }; + +} diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/ui/UiHandler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/ui/UiHandler.java new file mode 100644 index 0000000000000..a8b0b6002f5c4 --- /dev/null +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/ui/UiHandler.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.timeline.service.ui; + +import io.javalin.Javalin; + +/** + * Main User Interface (UI) Handler class that is responsible for rendering all UI pages. + */ +public class UiHandler { + + private final Javalin app; + + public UiHandler(Javalin app) { + this.app = app; + } + + public void register() { + app.get("/timeline", TimelineView.serveTimelineView); + } +} diff --git a/hudi-timeline-service/src/main/resources/templates/timeline.html b/hudi-timeline-service/src/main/resources/templates/timeline.html new file mode 100644 index 0000000000000..078e707c5dfa0 --- /dev/null +++ b/hudi-timeline-service/src/main/resources/templates/timeline.html @@ -0,0 +1,193 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + + + + + + Vis Timeline Example + + + + +

Hudi Timeline

+ +
+ + + +
+ + +
+ + +

+
+
+
+
+
+
+
diff --git a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestRequestHandler.java b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestRequestHandler.java
index e3838f3230023..023d7945b6179 100644
--- a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestRequestHandler.java
+++ b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestRequestHandler.java
@@ -27,6 +27,7 @@
 import org.apache.hudi.common.table.view.FileSystemViewStorageType;
 import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
 import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.metadata.HoodieTableMetadata;
 import org.apache.hudi.storage.StoragePath;
 import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
 import org.apache.hudi.timeline.TimelineServiceClient;
diff --git a/pom.xml b/pom.xml
index db9a20beb2734..4554c65d272b0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -187,6 +187,7 @@
     log4j2-surefire.properties
     0.13.0
     4.6.7
+    3.0.15.RELEASE
     9.4.57.v20241219
     3.1.0-incubating
     2.4.13