Skip to content

[WIP] HDFS-14801. PrometheusMetricsSink: Better support for NNTop. #1392

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
aajisaka wants to merge 2 commits into from
Closed

[WIP] HDFS-14801. PrometheusMetricsSink: Better support for NNTop. #1392

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cbf55e2
HDFS-14801. PrometheusMetricsSink: Better support for NNTop.
aajisaka Aug 30, 2019
f20da1b
Make PrometheusMetricsSink faster
aajisaka Sep 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 98 additions & 31 deletions ...ct/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
*/
package org.apache.hadoop.metrics2.sink;

import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.configuration2.SubsetConfiguration;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricType;
Expand All @@ -28,6 +29,7 @@
import java.io.Writer;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
Expand All @@ -47,6 +49,8 @@ public class PrometheusMetricsSink implements MetricsSink {
private static final Pattern SPLIT_PATTERN =
Pattern.compile("(?<!(^|[A-Z_]))(?=[A-Z])|(?<!^)(?=[A-Z][a-z])");
private static final Pattern DELIMITERS = Pattern.compile("[^a-zA-Z0-9]+");
private static final Pattern OP = Pattern.compile("op=([a-zA-Z*]+)\\.");
private static final Pattern USER = Pattern.compile("user=(.+)\\.count");

public PrometheusMetricsSink() {
}
Expand All @@ -56,39 +60,102 @@ public void putMetrics(MetricsRecord metricsRecord) {
for (AbstractMetric metrics : metricsRecord.metrics()) {
if (metrics.type() == MetricType.COUNTER
|| metrics.type() == MetricType.GAUGE) {

String key = prometheusName(
metricsRecord.name(), metrics.name());

StringBuilder builder = new StringBuilder();
builder.append("# TYPE ")
.append(key)
.append(" ")
.append(metrics.type().toString().toLowerCase())
.append("\n")
.append(key)
.append("{");
String sep = "";

//add tags
for (MetricsTag tag : metricsRecord.tags()) {
String tagName = tag.name().toLowerCase();

//ignore specific tag which includes sub-hierarchy
if (!tagName.equals("numopenconnectionsperuser")) {
builder.append(sep)
.append(tagName)
.append("=\"")
.append(tag.value())
.append("\"");
sep = ",";
}
}
builder.append("} ");
builder.append(metrics.value());
builder.append("\n");
final StringBuilder builder = new StringBuilder();
String key = buildMetrics(builder, metricsRecord, metrics);
metricLines.put(key, builder.toString());
}
}
}

private String buildMetrics(StringBuilder builder,
MetricsRecord metricsRecord,
AbstractMetric metrics) {
boolean isNNTopMetrics = false;
final String recordName = metricsRecord.name();

String key;
// Move window_ms, op, user from metrics name to metrics tag
if (recordName.startsWith("NNTopUserOpCounts")) {
isNNTopMetrics = true;
key = "nn_top_user_op_counts";
} else {
key = prometheusName(recordName, metrics.name());
}

builder.append("# TYPE ")
.append(key)
.append(" ")
.append(metrics.type().toString().toLowerCase())
.append("\n")
.append(key)
.append("{");
String sep = "";

//add tags
for (MetricsTag tag : metricsRecord.tags()) {
String tagName = tag.name().toLowerCase();
//ignore specific tag which includes sub-hierarchy
if (tagName.equals("numopenconnectionsperuser")) {
continue;
}
builder.append(sep)
.append(tagName)
.append("=\"")
.append(tag.value())
.append("\"");
sep = ",";
}

if (isNNTopMetrics) {
moveNNTopMetricsNameToLabels(builder, recordName, metrics.name());
}

builder.append("} ");
builder.append(metrics.value());
builder.append("\n");

if (isNNTopMetrics) {
return prometheusName(recordName, metrics.name());
} else {
return key;
}
}

/**
* Publish window_ms, op, and user from metrics name as metrics labels
* for better support of Prometheus.
*/
@VisibleForTesting
void moveNNTopMetricsNameToLabels(
StringBuilder builder, String recordName, String metricsName) {
final char sep = ',';
// Get window_ms
final int pos = recordName.indexOf("=");
final String window = recordName.substring(pos+1);
builder.append(sep)
.append("window_ms=\"")
.append(window)
.append('"');

// Get op
final Matcher opMatcher = OP.matcher(metricsName);
if (opMatcher.find()) {
final String op = opMatcher.group(1);
builder.append(sep)
.append("op=\"")
.append(op)
.append('"');
}

if (metricsName.endsWith(".count")) {
// Get user
final Matcher userMatcher = USER.matcher(metricsName);
if (userMatcher.find()) {
final String user = userMatcher.group(1);
builder.append(sep)
.append("user=\"")
.append(user)
.append('"');
}
}
}
Expand Down
27 changes: 27 additions & 0 deletions ...adoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestPrometheusMetricsSink.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;

import org.assertj.core.api.Assertions;
import org.junit.Assert;
import org.junit.Test;

Expand Down Expand Up @@ -121,6 +122,32 @@ public void testNamingWhitespaces() {
sink.prometheusName(recordName, metricName));
}

@Test
public void testMoveNNTopMetricsNameToLabels() {
PrometheusMetricsSink sink = new PrometheusMetricsSink();
// op total metric
String recordName = "NNTopUserOpCounts.windowMs=100000";
String metricName = "op=liststatus.TotalCount";
StringBuilder builder = new StringBuilder();
sink.moveNNTopMetricsNameToLabels(builder, recordName, metricName);
Assertions.assertThat(builder.toString())
.isEqualTo(",window_ms=\"100000\",op=\"liststatus\"");

builder.setLength(0);

// op user metric
metricName = "op=*.user=alice.count";
sink.moveNNTopMetricsNameToLabels(builder, recordName, metricName);
Assertions.assertThat(builder.toString())
.isEqualTo(",window_ms=\"100000\",op=\"*\",user=\"alice\"");

builder.setLength(0);
metricName = "op=listStatus.user=alice.count";
sink.moveNNTopMetricsNameToLabels(builder, recordName, metricName);
Assertions.assertThat(builder.toString())
.isEqualTo(",window_ms=\"100000\",op=\"listStatus\",user=\"alice\"");
}

/**
* Example metric pojo.
*/
Expand Down