HDDS-4196. Add an endpoint in Recon to query Prometheus

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java

@@ -41,4 +41,8 @@ private ReconConfigKeys() {
   public static final String OZONE_RECON_DATANODE_BIND_HOST_DEFAULT =
       "0.0.0.0";
   public static final int OZONE_RECON_DATANODE_PORT_DEFAULT = 9891;
+  // Prometheus HTTP endpoint including port
+  // ex: http://localhost:9090
+  public static final String OZONE_RECON_PROMETHEUS_HTTP_ENDPOINT =
+      "ozone.recon.prometheus.http.endpoint";
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -68,6 +68,8 @@ private void addPropertiesNotInXml() {
         OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE,
         OzoneConfigKeys.OZONE_S3_AUTHINFO_MAX_LIFETIME_KEY,
         ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR,
+        ReconServerConfigKeys.RECON_METRICS_HTTP_CONNECTION_TIMEOUT,
+        ReconServerConfigKeys.RECON_METRICS_HTTP_CONNECTION_REQUEST_TIMEOUT,
         OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY
         // TODO HDDS-2856
     ));

hadoop-ozone/pom.xml

@@ -267,7 +267,7 @@
             <exclude>**/output.xml</exclude>
             <exclude>**/log.html</exclude>
             <exclude>**/report.html</exclude>
-            <exclude>.idea/**</exclude>
+            <exclude>**/.idea/**</exclude>
             <exclude>**/.ssh/id_rsa*</exclude>
             <exclude>dev-support/*tests</exclude>
             <exclude>dev-support/checkstyle*</exclude>
@@ -304,6 +304,7 @@
             <exclude>**/yarn.lock</exclude>
             <exclude>**/pnpm-lock.yaml</exclude>
             <exclude>**/ozone-recon-web/build/**</exclude>
+            <exclude>src/test/resources/prometheus-test-response.txt</exclude>
             <exclude>src/main/license/**</exclude>
             <exclude>src/main/resources/proto.lock</exclude>
           </excludes>

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/MetricsServiceProviderFactory.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.recon.ReconConfigKeys;
+import org.apache.hadoop.ozone.recon.spi.MetricsServiceProvider;
+import org.apache.hadoop.ozone.recon.spi.impl.PrometheusServiceProviderImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+/**
+ * Factory class that is used to get the instance of configured Metrics Service
+ * Provider.
+ */
+@Singleton
+public class MetricsServiceProviderFactory {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(MetricsServiceProviderFactory.class);
+
+  private OzoneConfiguration configuration;
+  private ReconUtils reconUtils;
+
+  @Inject
+  public MetricsServiceProviderFactory(OzoneConfiguration configuration,
+                                       ReconUtils reconUtils) {
+    this.configuration = configuration;
+    this.reconUtils = reconUtils;
+  }
+
+  /**
+   * Returns the configured MetricsServiceProvider implementation (defaults
+   * to prometheus).
+   * If no metrics service providers are configured, returns null.
+   *
+   * @return MetricsServiceProvider instance that is configured.
+   */
+  public MetricsServiceProvider getMetricsServiceProvider() {
+    String prometheusEndpoint = getPrometheusEndpoint();
+    if (StringUtils.isNotEmpty(prometheusEndpoint)) {
+      if (LOG.isInfoEnabled()) {
+        LOG.info(
+            String.format("Choosing Prometheus as Metrics service provider " +
+                "with configured endpoint: %s", prometheusEndpoint));
+      }
+      return new PrometheusServiceProviderImpl(configuration, reconUtils);
+    }
+    return null;
+  }
+
+  /**
+   * Returns the Prometheus endpoint if configured. Otherwise returns null.
+   *
+   * @return Prometheus endpoint if configured, null otherwise.
+   */
+  private String getPrometheusEndpoint() {
+    String endpoint = configuration.getTrimmed(
+        ReconConfigKeys.OZONE_RECON_PROMETHEUS_HTTP_ENDPOINT);
+    // Remove the trailing slash from endpoint url.
+    if (endpoint != null && endpoint.endsWith("/")) {
+      endpoint = endpoint.substring(0, endpoint.length() - 1);
+    }
+    return endpoint;
+  }
+}

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java

@@ -108,6 +108,7 @@ protected void configure() {
         .to(StorageContainerServiceProviderImpl.class).in(Singleton.class);
     bind(OzoneStorageContainerManager.class)
         .to(ReconStorageContainerManagerFacade.class).in(Singleton.class);
+    bind(MetricsServiceProviderFactory.class).in(Singleton.class);
   }
 
   static class ReconOmTaskBindingModule extends AbstractModule {

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java

@@ -96,7 +96,17 @@ public final class ReconServerConfigKeys {
   public static final String OZONE_RECON_HTTP_AUTH_TYPE =
       OZONE_RECON_HTTP_AUTH_CONFIG_PREFIX + "type";
 
+  public static final String RECON_METRICS_HTTP_CONNECTION_TIMEOUT =
+      "ozone.recon.metrics.http.connection.timeout";
 
+  public static final String RECON_METRICS_HTTP_CONNECTION_TIMEOUT_DEFAULT =
+      "10s";
+
+  public static final String RECON_METRICS_HTTP_CONNECTION_REQUEST_TIMEOUT =
+      "ozone.recon.metrics.http.connection.request.timeout";
+
+  public static final String
+      RECON_METRICS_HTTP_CONNECTION_REQUEST_TIMEOUT_DEFAULT = "10s";
 
   /**
    * Private constructor for utility class.

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java

@@ -24,14 +24,14 @@
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
+import java.net.HttpURLConnection;
 import java.net.URL;
-import java.net.URLConnection;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.sql.Timestamp;
 import java.util.zip.GZIPOutputStream;
 
+import com.google.inject.Singleton;
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
@@ -59,6 +59,7 @@
 /**
  * Recon Utility class.
  */
+@Singleton
 public class ReconUtils {
 
   private final static int WRITE_BUFFER = 1048576; //1MB
@@ -222,20 +223,20 @@ public void untarCheckpointFile(File tarFile, Path destPath)
   }
 
   /**
-   * Make HTTP GET call on the URL and return inputstream to the response.
+   * Make HTTP GET call on the URL and return HttpURLConnection instance.
    * @param connectionFactory URLConnectionFactory to use.
    * @param url url to call
    * @param isSpnego is SPNEGO enabled
-   * @return Inputstream to the response of the HTTP call.
+   * @return HttpURLConnection instance of the HTTP call.
    * @throws IOException, AuthenticationException While reading the response.
    */
-  public InputStream makeHttpCall(URLConnectionFactory connectionFactory,
+  public HttpURLConnection makeHttpCall(URLConnectionFactory connectionFactory,
                                   String url, boolean isSpnego)
       throws IOException, AuthenticationException {
-    URLConnection urlConnection =
+    HttpURLConnection urlConnection = (HttpURLConnection)
           connectionFactory.openConnection(new URL(url), isSpnego);
     urlConnection.connect();
-    return urlConnection.getInputStream();
+    return urlConnection;
   }
 
   /**

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/MetricsProxyEndpoint.java

@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.api;
+
+import org.apache.hadoop.ozone.recon.MetricsServiceProviderFactory;
+import org.apache.hadoop.ozone.recon.spi.MetricsServiceProvider;
+import org.apache.hadoop.ozone.recon.spi.impl.PrometheusServiceProviderImpl;
+
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriInfo;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+
+import static org.apache.hadoop.ozone.recon.spi.impl.PrometheusServiceProviderImpl.PROMETHEUS_INSTANT_QUERY_API;
+
+
+/**
+ * Endpoint to fetch metrics data from Prometheus HTTP endpoint.
+ */
+@Path("/metrics")
+public class MetricsProxyEndpoint {
+
+  private MetricsServiceProvider metricsServiceProvider;
+
+  @Inject
+  public MetricsProxyEndpoint(
+      MetricsServiceProviderFactory metricsServiceProviderFactory) {
+    this.metricsServiceProvider =
+        metricsServiceProviderFactory.getMetricsServiceProvider();
+  }
+
+  /**
+   * Return a response from the configured metrics endpoint.
+   */
+  @GET
+  @Produces(MediaType.APPLICATION_JSON)
+  @Path("/{api}")
+  public void getMetricsResponse(
+      @DefaultValue (PROMETHEUS_INSTANT_QUERY_API) @PathParam("api") String api,
+      @Context UriInfo uriInfo,
+      @Context HttpServletResponse httpServletResponse
+  ) throws Exception {
+    if (metricsServiceProvider != null) {
+      HttpURLConnection connection = metricsServiceProvider.getMetricsResponse(
+          api, uriInfo.getRequestUri().getQuery());
+      InputStream inputStream;
+      if (Response.Status.fromStatusCode(connection.getResponseCode())
+          .getFamily() == Response.Status.Family.SUCCESSFUL) {
+        inputStream = connection.getInputStream();
+      } else {
+        // Throw a bad gateway error if HttpResponseCode is not 2xx
+        httpServletResponse.setStatus(HttpServletResponse.SC_BAD_GATEWAY);
+        inputStream = connection.getErrorStream();
+      }
+      try (
+          OutputStream outputStream =
+              httpServletResponse.getOutputStream();
+          ReadableByteChannel inputChannel =
+              Channels.newChannel(inputStream);
+          WritableByteChannel outputChannel =
+              Channels.newChannel(outputStream)
+        ) {
+        final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
+
+        while(inputChannel.read(buffer) != -1) {
+          buffer.flip();
+          outputChannel.write(buffer);
+          buffer.compact();
+        }
+
+        buffer.flip();
+
+        while(buffer.hasRemaining()) {
+          outputChannel.write(buffer);
+        }
+      } finally {
+        inputStream.close();
+      }
+    } else {
+      // Throw a Bad Gateway Error
+      httpServletResponse.sendError(HttpServletResponse.SC_BAD_GATEWAY,
+          "Metrics endpoint is not configured. Configure " +
+              PrometheusServiceProviderImpl.getEndpointConfigKey() + " and " +
+              "try again later.");
+    }
+  }
+}