diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
index c0b1d5d9386a..cbfa64c08539 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
@@ -41,4 +41,8 @@ private ReconConfigKeys() {
public static final String OZONE_RECON_DATANODE_BIND_HOST_DEFAULT =
"0.0.0.0";
public static final int OZONE_RECON_DATANODE_PORT_DEFAULT = 9891;
+ // Prometheus HTTP endpoint including port
+ // ex: http://localhost:9090
+ public static final String OZONE_RECON_PROMETHEUS_HTTP_ENDPOINT =
+ "ozone.recon.prometheus.http.endpoint";
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index cbd5db505f38..dec0c9c5067f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -68,6 +68,8 @@ private void addPropertiesNotInXml() {
OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE,
OzoneConfigKeys.OZONE_S3_AUTHINFO_MAX_LIFETIME_KEY,
ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR,
+ ReconServerConfigKeys.RECON_METRICS_HTTP_CONNECTION_TIMEOUT,
+ ReconServerConfigKeys.RECON_METRICS_HTTP_CONNECTION_REQUEST_TIMEOUT,
OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY
// TODO HDDS-2856
));
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 6f4b2b2e82ea..f5dccb6eced5 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -267,7 +267,7 @@
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.recon.ReconConfigKeys; +import org.apache.hadoop.ozone.recon.spi.MetricsServiceProvider; +import org.apache.hadoop.ozone.recon.spi.impl.PrometheusServiceProviderImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.inject.Singleton; + +/** + * Factory class that is used to get the instance of configured Metrics Service + * Provider. + */ +@Singleton +public class MetricsServiceProviderFactory { + + private static final Logger LOG = + LoggerFactory.getLogger(MetricsServiceProviderFactory.class); + + private OzoneConfiguration configuration; + private ReconUtils reconUtils; + + @Inject + public MetricsServiceProviderFactory(OzoneConfiguration configuration, + ReconUtils reconUtils) { + this.configuration = configuration; + this.reconUtils = reconUtils; + } + + /** + * Returns the configured MetricsServiceProvider implementation (defaults + * to prometheus). + * If no metrics service providers are configured, returns null. + * + * @return MetricsServiceProvider instance that is configured. + */ + public MetricsServiceProvider getMetricsServiceProvider() { + String prometheusEndpoint = getPrometheusEndpoint(); + if (StringUtils.isNotEmpty(prometheusEndpoint)) { + if (LOG.isInfoEnabled()) { + LOG.info( + String.format("Choosing Prometheus as Metrics service provider " + + "with configured endpoint: %s", prometheusEndpoint)); + } + return new PrometheusServiceProviderImpl(configuration, reconUtils); + } + return null; + } + + /** + * Returns the Prometheus endpoint if configured. Otherwise returns null. + * + * @return Prometheus endpoint if configured, null otherwise. + */ + private String getPrometheusEndpoint() { + String endpoint = configuration.getTrimmed( + ReconConfigKeys.OZONE_RECON_PROMETHEUS_HTTP_ENDPOINT); + // Remove the trailing slash from endpoint url. + if (endpoint != null && endpoint.endsWith("/")) { + endpoint = endpoint.substring(0, endpoint.length() - 1); + } + return endpoint; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java index 89917675a174..cb667f43855b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java @@ -108,6 +108,7 @@ protected void configure() { .to(StorageContainerServiceProviderImpl.class).in(Singleton.class); bind(OzoneStorageContainerManager.class) .to(ReconStorageContainerManagerFacade.class).in(Singleton.class); + bind(MetricsServiceProviderFactory.class).in(Singleton.class); } static class ReconOmTaskBindingModule extends AbstractModule { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java index 704c18e18357..78e805151c7d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java @@ -96,7 +96,17 @@ public final class ReconServerConfigKeys { public static final String OZONE_RECON_HTTP_AUTH_TYPE = OZONE_RECON_HTTP_AUTH_CONFIG_PREFIX + "type"; + public static final String RECON_METRICS_HTTP_CONNECTION_TIMEOUT = + "ozone.recon.metrics.http.connection.timeout"; + public static final String RECON_METRICS_HTTP_CONNECTION_TIMEOUT_DEFAULT = + "10s"; + + public static final String RECON_METRICS_HTTP_CONNECTION_REQUEST_TIMEOUT = + "ozone.recon.metrics.http.connection.request.timeout"; + + public static final String + RECON_METRICS_HTTP_CONNECTION_REQUEST_TIMEOUT_DEFAULT = "10s"; /** * Private constructor for utility class. diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index b6e4d7c0f179..752cbce10bb3 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -24,14 +24,14 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; +import java.net.HttpURLConnection; import java.net.URL; -import java.net.URLConnection; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Timestamp; import java.util.zip.GZIPOutputStream; +import com.google.inject.Singleton; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; @@ -59,6 +59,7 @@ /** * Recon Utility class. */ +@Singleton public class ReconUtils { private final static int WRITE_BUFFER = 1048576; //1MB @@ -222,20 +223,20 @@ public void untarCheckpointFile(File tarFile, Path destPath) } /** - * Make HTTP GET call on the URL and return inputstream to the response. + * Make HTTP GET call on the URL and return HttpURLConnection instance. * @param connectionFactory URLConnectionFactory to use. * @param url url to call * @param isSpnego is SPNEGO enabled - * @return Inputstream to the response of the HTTP call. + * @return HttpURLConnection instance of the HTTP call. * @throws IOException, AuthenticationException While reading the response. */ - public InputStream makeHttpCall(URLConnectionFactory connectionFactory, + public HttpURLConnection makeHttpCall(URLConnectionFactory connectionFactory, String url, boolean isSpnego) throws IOException, AuthenticationException { - URLConnection urlConnection = + HttpURLConnection urlConnection = (HttpURLConnection) connectionFactory.openConnection(new URL(url), isSpnego); urlConnection.connect(); - return urlConnection.getInputStream(); + return urlConnection; } /** diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/MetricsProxyEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/MetricsProxyEndpoint.java new file mode 100644 index 000000000000..327e9b192663 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/MetricsProxyEndpoint.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.api;
+
+import org.apache.hadoop.ozone.recon.MetricsServiceProviderFactory;
+import org.apache.hadoop.ozone.recon.spi.MetricsServiceProvider;
+import org.apache.hadoop.ozone.recon.spi.impl.PrometheusServiceProviderImpl;
+
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriInfo;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+
+import static org.apache.hadoop.ozone.recon.spi.impl.PrometheusServiceProviderImpl.PROMETHEUS_INSTANT_QUERY_API;
+
+
+/**
+ * Endpoint to fetch metrics data from Prometheus HTTP endpoint.
+ */
+@Path("/metrics")
+public class MetricsProxyEndpoint {
+
+ private MetricsServiceProvider metricsServiceProvider;
+
+ @Inject
+ public MetricsProxyEndpoint(
+ MetricsServiceProviderFactory metricsServiceProviderFactory) {
+ this.metricsServiceProvider =
+ metricsServiceProviderFactory.getMetricsServiceProvider();
+ }
+
+ /**
+ * Return a response from the configured metrics endpoint.
+ */
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ @Path("/{api}")
+ public void getMetricsResponse(
+ @DefaultValue (PROMETHEUS_INSTANT_QUERY_API) @PathParam("api") String api,
+ @Context UriInfo uriInfo,
+ @Context HttpServletResponse httpServletResponse
+ ) throws Exception {
+ if (metricsServiceProvider != null) {
+ HttpURLConnection connection = metricsServiceProvider.getMetricsResponse(
+ api, uriInfo.getRequestUri().getQuery());
+ InputStream inputStream;
+ if (Response.Status.fromStatusCode(connection.getResponseCode())
+ .getFamily() == Response.Status.Family.SUCCESSFUL) {
+ inputStream = connection.getInputStream();
+ } else {
+ // Throw a bad gateway error if HttpResponseCode is not 2xx
+ httpServletResponse.setStatus(HttpServletResponse.SC_BAD_GATEWAY);
+ inputStream = connection.getErrorStream();
+ }
+ try (
+ OutputStream outputStream =
+ httpServletResponse.getOutputStream();
+ ReadableByteChannel inputChannel =
+ Channels.newChannel(inputStream);
+ WritableByteChannel outputChannel =
+ Channels.newChannel(outputStream)
+ ) {
+ final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
+
+ while(inputChannel.read(buffer) != -1) {
+ buffer.flip();
+ outputChannel.write(buffer);
+ buffer.compact();
+ }
+
+ buffer.flip();
+
+ while(buffer.hasRemaining()) {
+ outputChannel.write(buffer);
+ }
+ } finally {
+ inputStream.close();
+ }
+ } else {
+ // Throw a Bad Gateway Error
+ httpServletResponse.sendError(HttpServletResponse.SC_BAD_GATEWAY,
+ "Metrics endpoint is not configured. Configure " +
+ PrometheusServiceProviderImpl.getEndpointConfigKey() + " and " +
+ "try again later.");
+ }
+ }
+}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/PipelineEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/PipelineEndpoint.java
index d75469719388..045cdc0b3baa 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/PipelineEndpoint.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/PipelineEndpoint.java
@@ -20,9 +20,12 @@
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
+import org.apache.hadoop.ozone.recon.MetricsServiceProviderFactory;
import org.apache.hadoop.ozone.recon.api.types.PipelineMetadata;
import org.apache.hadoop.ozone.recon.api.types.PipelinesResponse;
+import org.apache.hadoop.ozone.recon.metrics.Metric;
import org.apache.hadoop.ozone.recon.scm.ReconPipelineManager;
+import org.apache.hadoop.ozone.recon.spi.MetricsServiceProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,6 +39,8 @@
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
+import java.util.TreeMap;
import java.util.UUID;
/**
@@ -49,10 +54,15 @@ public class PipelineEndpoint {
LoggerFactory.getLogger(PipelineEndpoint.class);
private ReconPipelineManager pipelineManager;
+ private MetricsServiceProvider metricsServiceProvider;
@Inject
- PipelineEndpoint(OzoneStorageContainerManager reconSCM) {
+ PipelineEndpoint(OzoneStorageContainerManager reconSCM,
+ MetricsServiceProviderFactory
+ metricsServiceProviderFactory) {
this.pipelineManager = (ReconPipelineManager) reconSCM.getPipelineManager();
+ this.metricsServiceProvider =
+ metricsServiceProviderFactory.getMetricsServiceProvider();
}
/**
@@ -89,19 +99,58 @@ public Response getPipelines() {
LOG.warn("Cannot get containers for pipeline {} ", pipelineId, ioEx);
}
- PipelineMetadata pipelineMetadata = builder.setPipelineId(pipelineId)
+ PipelineMetadata.Builder pipelineBuilder =
+ builder.setPipelineId(pipelineId)
.setDatanodes(datanodes)
.setDuration(duration)
.setStatus(pipeline.getPipelineState())
.setReplicationFactor(pipeline.getFactor().getNumber())
- .setReplicationType(pipeline.getType().toString())
- .build();
+ .setReplicationType(pipeline.getType().toString());
- pipelinesList.add(pipelineMetadata);
+ // If any metrics service providers like Prometheus
+ // is configured, then query it for metrics and populate
+ // leader election count and last leader election time
+ if (metricsServiceProvider != null) {
+ // Extract last part of pipelineId to get its group Id.
+ // ex. group id of 48981bf7-8bea-4fbd-9857-79df51ee872d
+ // is group-79DF51EE872D
+ String[] splits = pipelineId.toString().split("-");
+ String groupId = "group-" + splits[splits.length-1].toUpperCase();
+ Optional
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.spi;
+
+import org.apache.hadoop.ozone.recon.metrics.Metric;
+
+import java.net.HttpURLConnection;
+import java.util.List;
+
+/**
+ * Interface to access Ozone metrics.
+ */
+public interface MetricsServiceProvider {
+
+ /**
+ * Returns {@link HttpURLConnection} after querying Metrics endpoint for the
+ * given metric.
+ *
+ * @param api api.
+ * @param queryString query string with metric name and other filters.
+ * @return HttpURLConnection
+ * @throws Exception exception
+ */
+ HttpURLConnection getMetricsResponse(String api, String queryString)
+ throws Exception;
+
+ /**
+ * Returns a list of {@link Metric} for the given instant query.
+ *
+ * @param queryString query string with metric name and other filters.
+ * @return List of Json map of metrics response.
+ * @throws Exception exception
+ */
+ List
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.spi.impl;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.recon.ReconConfigKeys;
+import org.apache.hadoop.hdfs.web.URLConnectionFactory;
+import org.apache.hadoop.ozone.recon.ReconUtils;
+import org.apache.hadoop.ozone.recon.metrics.Metric;
+import org.apache.hadoop.ozone.recon.spi.MetricsServiceProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Singleton;
+import javax.ws.rs.core.Response;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_METRICS_HTTP_CONNECTION_REQUEST_TIMEOUT;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_METRICS_HTTP_CONNECTION_REQUEST_TIMEOUT_DEFAULT;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_METRICS_HTTP_CONNECTION_TIMEOUT;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_METRICS_HTTP_CONNECTION_TIMEOUT_DEFAULT;
+
+/**
+ * Implementation of the Prometheus Metrics Service provider.
+ */
+@Singleton
+public class PrometheusServiceProviderImpl
+ implements MetricsServiceProvider {
+
+ public static final String PROMETHEUS_INSTANT_QUERY_API = "query";
+ public static final String PROMETHEUS_RANGED_QUERY_API = "query_range";
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PrometheusServiceProviderImpl.class);
+
+ private URLConnectionFactory connectionFactory;
+ private final String prometheusEndpoint;
+ private ReconUtils reconUtils;
+
+ public PrometheusServiceProviderImpl(OzoneConfiguration configuration,
+ ReconUtils reconUtils) {
+
+ int connectionTimeout = (int) configuration.getTimeDuration(
+ RECON_METRICS_HTTP_CONNECTION_TIMEOUT,
+ RECON_METRICS_HTTP_CONNECTION_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
+ int connectionRequestTimeout = (int) configuration.getTimeDuration(
+ RECON_METRICS_HTTP_CONNECTION_REQUEST_TIMEOUT,
+ RECON_METRICS_HTTP_CONNECTION_REQUEST_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
+
+ connectionFactory =
+ URLConnectionFactory.newDefaultURLConnectionFactory(connectionTimeout,
+ connectionRequestTimeout, configuration);
+
+ String endpoint = configuration.getTrimmed(getEndpointConfigKey());
+ // Remove the trailing slash from endpoint url.
+ if (endpoint != null && endpoint.endsWith("/")) {
+ endpoint = endpoint.substring(0, endpoint.length() - 1);
+ }
+ this.prometheusEndpoint = endpoint;
+ this.reconUtils = reconUtils;
+ }
+
+ /**
+ * Returns {@link HttpURLConnection} after querying Metrics endpoint for the
+ * given metric.
+ *
+ * @param api api.
+ * @param queryString query string with metric name and other filters.
+ * @return HttpURLConnection
+ * @throws Exception exception
+ */
+ @Override
+ public HttpURLConnection getMetricsResponse(String api, String queryString)
+ throws Exception {
+ String url = String.format("%s/api/v1/%s?%s", prometheusEndpoint, api,
+ queryString);
+ return reconUtils.makeHttpCall(connectionFactory,
+ url, false);
+ }
+
+ /**
+ * Returns the endpoint configuration key for the Metrics service provider.
+ *
+ * @return endpoint configuration key.
+ */
+ public static String getEndpointConfigKey() {
+ return ReconConfigKeys.OZONE_RECON_PROMETHEUS_HTTP_ENDPOINT;
+ }
+
+ /**
+ * Returns a list of {@link Metric} for the given instant query.
+ *
+ * @param queryString query string with metric name and other filters.
+ * @return List of Json map of metrics response.
+ * @throws Exception exception
+ */
+ @Override
+ public List