From de9fa0ca2163b6a1c8cf43888a1f02ec1e555c84 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 17 May 2025 19:58:11 +0800 Subject: [PATCH 1/3] HADOOP-19571. Improve PrometheusMetricsSink#normalizeName performance --- .../metrics2/sink/PrometheusMetricsSink.java | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java index 9024203700ee1..553e979e8a1de 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java @@ -19,7 +19,13 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutionException; import java.util.regex.Matcher; + +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.UncheckedExecutionException; import org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricType; @@ -35,6 +41,8 @@ import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Metrics sink for prometheus exporter. @@ -42,6 +50,7 @@ * Stores the metric data in-memory and return with it on request. */ public class PrometheusMetricsSink implements MetricsSink { + private static final Logger LOG = LoggerFactory.getLogger(PrometheusMetricsSink.class); /** * Cached output lines for each metrics. @@ -62,6 +71,17 @@ public class PrometheusMetricsSink implements MetricsSink { Pattern .compile("^op=(?\\w+)(.user=(?.*)|)\\.(TotalCount|count)$"); + /** + * A fixed cache for Hadoop metric to Prometheus metric name conversion. + */ + private static final int NORMALIZED_NAME_CACHE_MAX_SIZE = 100_000; + private static final CacheLoader NORMALIZED_NAME_CACHE_LOADER = + CacheLoader.from(PrometheusMetricsSink::normalizeImpl); + private static final LoadingCache NORMALIZED_NAME_CACHE = + CacheBuilder.newBuilder() + .maximumSize(NORMALIZED_NAME_CACHE_MAX_SIZE) + .build(NORMALIZED_NAME_CACHE_LOADER); + public PrometheusMetricsSink() { } @@ -83,7 +103,22 @@ public void putMetrics(MetricsRecord metricsRecord) { /** * Convert CamelCase based names to lower-case names where the separator - * is the underscore, to follow prometheus naming conventions. + * is the underscore, to follow prometheus naming conventions. This method + * utilizes a cache to improve performance. + * + *

+ * Reference: + *

+ *

* * @param metricName metricName. * @param recordName recordName. @@ -93,6 +128,22 @@ public String prometheusName(String recordName, String metricName) { String baseName = StringUtils.capitalize(recordName) + StringUtils.capitalize(metricName); + try { + return NORMALIZED_NAME_CACHE.get(baseName); + } catch (ExecutionException | UncheckedExecutionException e) { + // This should not happen since normalization function do not throw any exception + // Nevertheless, we can fall back to uncached implementation if it somehow happens. + LOG.warn("Exception encountered when loading metric with base name {} from cache, " + + "fall back to uncached normalization implementation", baseName, e); + return normalizeImpl(baseName); + } + } + + /** + * Underlying Prometheus normalization implementation. + * See {@link PrometheusMetricsSink#prometheusName(String, String)} for more information. + */ + private static String normalizeImpl(String baseName) { String[] parts = SPLIT_PATTERN.split(baseName); String joined = String.join("_", parts).toLowerCase(); return DELIMITERS.matcher(joined).replaceAll("_"); From ba85667767669ec53779219032490e1d0a8efabc Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Mon, 19 May 2025 09:34:43 +0800 Subject: [PATCH 2/3] Fix javadoc --- .../org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java index 553e979e8a1de..b008fc5177374 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java @@ -118,7 +118,6 @@ public void putMetrics(MetricsRecord metricsRecord) { * Exposition formats * * - *

* * @param metricName metricName. * @param recordName recordName. @@ -138,7 +137,7 @@ public String prometheusName(String recordName, return normalizeImpl(baseName); } } - + /** * Underlying Prometheus normalization implementation. * See {@link PrometheusMetricsSink#prometheusName(String, String)} for more information. From ad21951b9694ad5d704881147ce72ea9360fc771 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Thu, 22 May 2025 13:29:18 +0800 Subject: [PATCH 3/3] Trigger Build