From 347641d0cef968cb368ec7ebd073316026a82af7 Mon Sep 17 00:00:00 2001 From: anoop narang Date: Tue, 1 Feb 2022 10:24:55 +0530 Subject: [PATCH] [MINOR] Add configuration to enable/disable common prefix for all metrics --- .../org/apache/hudi/config/HoodieWriteConfig.java | 4 ++++ .../hudi/config/metrics/HoodieMetricsConfig.java | 12 ++++++++++++ .../metadata/HoodieBackedTableMetadataWriter.java | 1 + .../java/org/apache/hudi/metrics/HoodieMetrics.java | 8 +++++++- .../main/java/org/apache/hudi/metrics/Metrics.java | 4 ++-- .../deltastreamer/HoodieDeltaStreamerMetrics.java | 8 +++++++- 6 files changed, 33 insertions(+), 4 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index d8a928584ea09..7e718249a767b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -1561,6 +1561,10 @@ public boolean isExecutorMetricsEnabled() { getStringOrDefault(HoodieMetricsConfig.EXECUTOR_METRICS_ENABLE, "false")); } + public boolean isMetricsCommonPrefixEnabled() { + return getBooleanOrDefault(HoodieMetricsConfig.ENABLE_COMMON_PREFIX); + } + public MetricsReporterType getMetricsReporterType() { return MetricsReporterType.valueOf(getString(HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE)); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java index 8396540394b86..3a1eedda8994b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java @@ -84,6 +84,13 @@ public class HoodieMetricsConfig extends HoodieConfig { .sinceVersion("0.7.0") .withDocumentation(""); + public static final ConfigProperty ENABLE_COMMON_PREFIX = ConfigProperty + .key(METRIC_PREFIX + ".common_prefix.enable") + .defaultValue(true) + .sinceVersion("0.10.0") + .withDocumentation("Turn on/off common prefix for all metrics. Table name is used as the common prefix and its " + + "on by default"); + /** * @deprecated Use {@link #TURN_METRICS_ON} and its methods instead */ @@ -164,6 +171,11 @@ public Builder withExecutorMetrics(boolean enable) { return this; } + public Builder withCommonPrefix(boolean enable) { + hoodieMetricsConfig.setValue(ENABLE_COMMON_PREFIX, String.valueOf(enable)); + return this; + } + public HoodieMetricsConfig build() { hoodieMetricsConfig.setDefaults(HoodieMetricsConfig.class.getName()); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 49e2c8f74a1ae..aff9195c34623 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -225,6 +225,7 @@ private HoodieWriteConfig createMetadataWriteConfig(HoodieWriteConfig writeConfi builder.withMetricsConfig(HoodieMetricsConfig.newBuilder() .withReporterType(writeConfig.getMetricsReporterType().toString()) .withExecutorMetrics(writeConfig.isExecutorMetricsEnabled()) + .withCommonPrefix(writeConfig.isMetricsCommonPrefixEnabled()) .on(true).build()); switch (writeConfig.getMetricsReporterType()) { case GRAPHITE: diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java index e874047b8c644..d30554b120aa4 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java @@ -215,7 +215,13 @@ public void updateIndexMetrics(final String action, final long durationInMs) { } String getMetricsName(String action, String metric) { - return config == null ? null : String.format("%s.%s.%s", config.getMetricReporterMetricsNamePrefix(), action, metric); + if (config == null) { + return null; + } + if (config.isMetricsCommonPrefixEnabled()) { + return String.format("%s.%s.%s", tableName, action, metric); + } + return String.format("%s.%s", action, metric); } /** diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/Metrics.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/Metrics.java index b570f512f371d..e6836cff71f67 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/Metrics.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/Metrics.java @@ -46,7 +46,7 @@ public class Metrics { private Metrics(HoodieWriteConfig metricConfig) { registry = new MetricRegistry(); - commonMetricPrefix = metricConfig.getMetricReporterMetricsNamePrefix(); + commonMetricPrefix = metricConfig.isMetricsCommonPrefixEnabled() ? metricConfig.getTableName() : ""; reporter = MetricsReporterFactory.createReporter(metricConfig, registry); if (reporter == null) { throw new RuntimeException("Cannot initialize Reporter."); @@ -116,7 +116,7 @@ public static synchronized void flush() { } public static void registerGauges(Map metricsMap, Option prefix) { - String metricPrefix = prefix.isPresent() ? prefix.get() + "." : ""; + String metricPrefix = prefix.isPresent() && !prefix.get().isEmpty() ? prefix.get() + "." : ""; metricsMap.forEach((k, v) -> registerGauge(metricPrefix + k, v)); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java index d361179a1db15..98dfc6a533873 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java @@ -74,7 +74,13 @@ private Timer createTimer(String name) { } String getMetricsName(String action, String metric) { - return config == null ? null : String.format("%s.%s.%s", config.getMetricReporterMetricsNamePrefix(), action, metric); + if (config == null) { + return null; + } + if (config.isMetricsCommonPrefixEnabled()) { + return String.format("%s.%s.%s", tableName, action, metric); + } + return String.format("%s.%s", action, metric); } public void updateDeltaStreamerMetrics(long durationInNs) {