Skip to content

[HUDI-4796] MetricsReporter stop bug #6619

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yihua merged 4 commits into from
Sep 16, 2022
Merged

[HUDI-4796] MetricsReporter stop bug #6619

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
15f925f
get rid of unnecessary method and make sure stop is called instead
the-other-tim-brown Sep 7, 2022
461a755
remove changes to unrelated file
the-other-tim-brown Sep 7, 2022
138acf4
Merge branch 'master' into HUDI-4796
the-other-tim-brown Sep 7, 2022
7e9a46c
rename method
the-other-tim-brown Sep 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 4 additions & 11 deletions ...ient/hudi-client-common/src/main/java/org/apache/hudi/metrics/ConsoleMetricsReporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,13 @@

package org.apache.hudi.metrics;

import java.io.Closeable;
import java.util.concurrent.TimeUnit;

import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;

import com.codahale.metrics.ConsoleReporter;
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;

import java.util.concurrent.TimeUnit;

/**
* Hudi Console metrics reporter. Reports the metrics by printing them to the stdout on the console.
Expand Down Expand Up @@ -61,11 +59,6 @@ public void report() {
}
}

@Override
public Closeable getReporter() {
return consoleReporter;
}

@Override
public void stop() {
if (consoleReporter != null) {
Expand Down
7 changes: 0 additions & 7 deletions ...ent/hudi-client-common/src/main/java/org/apache/hudi/metrics/InMemoryMetricsReporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@

package org.apache.hudi.metrics;

import java.io.Closeable;

/**
* Used for testing.
*/
Expand All @@ -31,11 +29,6 @@ public void start() {}
@Override
public void report() {}

@Override
public Closeable getReporter() {
return null;
}

@Override
public void stop() {

Expand Down
7 changes: 0 additions & 7 deletions hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,10 @@

import javax.management.MBeanServer;

import java.io.Closeable;
import java.lang.management.ManagementFactory;
import java.util.Objects;
import java.util.stream.IntStream;


/**
* Implementation of Jmx reporter, which used to report jmx metric.
*/
Expand Down Expand Up @@ -86,11 +84,6 @@ public void start() {
public void report() {
}

@Override
public Closeable getReporter() {
return jmxReporterServer.getReporter();
}

@Override
public void stop() {
Objects.requireNonNull(jmxReporterServer, "jmxReporterServer is not running.");
Expand Down
15 changes: 4 additions & 11 deletions hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/Metrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;

import java.io.Closeable;
import java.util.Map;

/**
Expand Down Expand Up @@ -56,14 +55,12 @@ private Metrics(HoodieWriteConfig metricConfig) {
Runtime.getRuntime().addShutdownHook(new Thread(Metrics::shutdown));
}

private void reportAndCloseReporter() {
private void reportAndStopReporter() {
try {
registerHoodieCommonMetrics();
reporter.report();
if (getReporter() != null) {
LOG.info("Closing metrics reporter...");
getReporter().close();
}
LOG.info("Stopping the metrics reporter...");
reporter.stop();
Copy link
Contributor

@nsivabalan nsivabalan Sep 15, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor. may be we can rename the method to renameAndStopReporter

} catch (Exception e) {
LOG.warn("Error while closing reporter", e);
}
Expand Down Expand Up @@ -105,7 +102,7 @@ public static synchronized void shutdown() {
if (!initialized) {
return;
}
instance.reportAndCloseReporter();
instance.reportAndStopReporter();
initialized = false;
}

Expand Down Expand Up @@ -137,10 +134,6 @@ public MetricRegistry getRegistry() {
return registry;
}

public Closeable getReporter() {
return reporter.getReporter();
}

public static boolean isInitialized() {
return initialized;
}
Expand Down
6 changes: 0 additions & 6 deletions ...ent/hudi-client-common/src/main/java/org/apache/hudi/metrics/MetricsGraphiteReporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;

import java.io.Closeable;
import java.net.InetSocketAddress;
import java.util.concurrent.TimeUnit;

Expand Down Expand Up @@ -78,11 +77,6 @@ public void report() {
}
}

@Override
public Closeable getReporter() {
return graphiteReporter;
}

private GraphiteReporter createGraphiteReport() {
Graphite graphite = new Graphite(new InetSocketAddress(serverHost, serverPort));
String reporterPrefix = config.getGraphiteMetricPrefix();
Expand Down
4 changes: 0 additions & 4 deletions hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/MetricsReporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@

package org.apache.hudi.metrics;

import java.io.Closeable;

/**
* Interface for implementing a Reporter.
*/
Expand All @@ -35,8 +33,6 @@ public abstract class MetricsReporter {
*/
public abstract void report();

public abstract Closeable getReporter();

/**
* Stop this reporter. Should be used to stop channels, streams and release resources.
*/
Expand Down
6 changes: 0 additions & 6 deletions ...nt-common/src/main/java/org/apache/hudi/metrics/cloudwatch/CloudWatchMetricsReporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;

import java.io.Closeable;
import java.util.concurrent.TimeUnit;

/**
Expand Down Expand Up @@ -72,11 +71,6 @@ public void report() {
reporter.report();
}

@Override
public Closeable getReporter() {
return reporter;
}

@Override
public void stop() {
LOG.info("Stopping CloudWatch Metrics Reporter.");
Expand Down
6 changes: 0 additions & 6 deletions ...i-client-common/src/main/java/org/apache/hudi/metrics/datadog/DatadogMetricsReporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;

import java.io.Closeable;
import java.util.List;
import java.util.concurrent.TimeUnit;

Expand Down Expand Up @@ -81,11 +80,6 @@ public void report() {
reporter.report();
}

@Override
public Closeable getReporter() {
return reporter;
}

@Override
public void stop() {
reporter.stop();
Expand Down
13 changes: 4 additions & 9 deletions ...di-client-common/src/main/java/org/apache/hudi/metrics/prometheus/PrometheusReporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,17 @@

package org.apache.hudi.metrics.prometheus;

import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.metrics.MetricsReporter;

import com.codahale.metrics.MetricRegistry;
import io.prometheus.client.CollectorRegistry;
import io.prometheus.client.dropwizard.DropwizardExports;
import io.prometheus.client.exporter.HTTPServer;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.metrics.MetricsReporter;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;

import java.io.Closeable;
import java.net.InetSocketAddress;

/**
Expand Down Expand Up @@ -65,11 +65,6 @@ public void start() {
public void report() {
}

@Override
public Closeable getReporter() {
return null;
}

@Override
public void stop() {
collectorRegistry.unregister(metricExports);
Expand Down
11 changes: 3 additions & 8 deletions ...t-common/src/main/java/org/apache/hudi/metrics/prometheus/PushGatewayMetricsReporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@

package org.apache.hudi.metrics.prometheus;

import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.metrics.MetricsReporter;

import java.io.Closeable;
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;

import java.util.Random;
import java.util.concurrent.TimeUnit;

Expand Down Expand Up @@ -65,11 +65,6 @@ public void report() {
pushGatewayReporter.report(null, null, null, null, null);
}

@Override
public Closeable getReporter() {
return pushGatewayReporter;
}

@Override
public void stop() {
pushGatewayReporter.stop();
Expand Down
6 changes: 0 additions & 6 deletions .../hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;

import java.io.Closeable;
import java.util.Properties;

import static org.junit.jupiter.api.Assertions.assertEquals;
Expand Down Expand Up @@ -87,11 +86,6 @@ public void start() {}
@Override
public void report() {}

@Override
public Closeable getReporter() {
return null;
}

@Override
public void stop() {}
}
Expand Down