From 48404f0214abfc089b24d9f66022258bea45c978 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 22 Jun 2021 14:26:29 +0200 Subject: [PATCH] [HUDI-2055] Added metric for time of lastSync --- .../org/apache/hudi/utilities/deltastreamer/DeltaSync.java | 2 ++ .../utilities/deltastreamer/HoodieDeltaStreamerMetrics.java | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java index c60449a79768..0e5afa617798 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java @@ -289,6 +289,8 @@ public Pair, JavaRDD> syncOnce() throws IOException srcRecordsWithCkpt.getRight().getLeft(), metrics, overallTimerContext); } + metrics.updateDeltaStreamerSyncMetrics(System.currentTimeMillis()); + // Clear persistent RDDs jssc.getPersistentRDDs().values().forEach(JavaRDD::unpersist); return result; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java index 056713771652..0c2f18fa4279 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java @@ -95,6 +95,12 @@ public void updateDeltaStreamerKafkaDelayCountMetrics(long kafkaDelayCount) { } } + public void updateDeltaStreamerSyncMetrics(long syncEpochTimeInMs) { + if (config.isMetricsOn()) { + Metrics.registerGauge(getMetricsName("deltastreamer", "lastSync"), syncEpochTimeInMs); + } + } + public long getDurationInMs(long ctxDuration) { return ctxDuration / 1000000; }