Merge pull request #3067 from Amila-Rukshan/jmx-monitoring

Jmx monitoring for enforcer

enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/ConfigHolder.java

@@ -68,6 +68,7 @@
 import org.wso2.choreo.connect.enforcer.config.dto.TracingDTO;
 import org.wso2.choreo.connect.enforcer.constants.APIConstants;
 import org.wso2.choreo.connect.enforcer.constants.Constants;
+import org.wso2.choreo.connect.enforcer.jmx.MBeanRegistrator;
 import org.wso2.choreo.connect.enforcer.throttle.databridge.agent.conf.AgentConfiguration;
 import org.wso2.choreo.connect.enforcer.util.BackendJwtUtils;
 import org.wso2.choreo.connect.enforcer.util.FilterUtils;
@@ -235,6 +236,8 @@ private void populateAuthService(Service cdsAuth) {
         authDto.setMaxMessageSize(cdsAuth.getMaxMessageSize());
 
         ThreadPoolConfig threadPool = new ThreadPoolConfig();
+        MBeanRegistrator.registerMBean(threadPool);
+
         threadPool.setCoreSize(cdsAuth.getThreadPool().getCoreSize());
         threadPool.setKeepAliveTime(cdsAuth.getThreadPool().getKeepAliveTime());
         threadPool.setMaxSize(cdsAuth.getThreadPool().getMaxSize());

enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/dto/ThreadPoolConfig.java

@@ -21,7 +21,7 @@
 /**
  * Holds the configurations related to threading of gRPC netty server.
  */
-public class ThreadPoolConfig {
+public class ThreadPoolConfig implements ThreadPoolConfigMBean {
     private int coreSize;
     private int maxSize;
     private int keepAliveTime;

enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/dto/ThreadPoolConfigMBean.java

@@ -0,0 +1,50 @@
+/*
+ *  Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.wso2.choreo.connect.enforcer.config.dto;
+
+/**
+ * MBean API for Thread Pool Configuration.
+ */
+public interface ThreadPoolConfigMBean {
+    /**
+     * Getter for core size.
+     * 
+     * @return int
+     */
+    public int getCoreSize();
+
+    /**
+     * Getter for max size.
+     * 
+     * @return int
+     */
+    public int getMaxSize();
+
+    /**
+     * Getter for keep alive size.
+     * 
+     * @return int
+     */
+    public int getKeepAliveTime();
+
+    /**
+     * Getter for queue size.
+     * 
+     * @return int
+     */
+    public int getQueueSize();
+}

enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/grpc/ExtAuthService.java

@@ -39,8 +39,10 @@
 import org.wso2.choreo.connect.enforcer.constants.HttpConstants;
 import org.wso2.choreo.connect.enforcer.constants.RouterAccessLogConstants;
 import org.wso2.choreo.connect.enforcer.deniedresponse.DeniedResponsePreparer;
+import org.wso2.choreo.connect.enforcer.jmx.JMXUtils;
 import org.wso2.choreo.connect.enforcer.metrics.MetricsExporter;
 import org.wso2.choreo.connect.enforcer.metrics.MetricsManager;
+import org.wso2.choreo.connect.enforcer.metrics.jmx.impl.ExtAuthMetrics;
 import org.wso2.choreo.connect.enforcer.server.HttpRequestHandler;
 import org.wso2.choreo.connect.enforcer.tracing.TracingConstants;
 import org.wso2.choreo.connect.enforcer.tracing.TracingContextHolder;
@@ -93,6 +95,9 @@ public void check(CheckRequest request, StreamObserver<CheckResponse> responseOb
                 MetricsExporter metricsExporter = MetricsManager.getInstance();
                 metricsExporter.trackMetric("enforcerLatency", System.currentTimeMillis() - starTimestamp);
             }
+            if (JMXUtils.isJMXMetricsEnabled()) {
+                ExtAuthMetrics.getInstance().recordMetric(System.currentTimeMillis() - starTimestamp);
+            }
         }
     }
 

enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/JMXAgent.java

@@ -0,0 +1,66 @@
+/*
+ *  Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.wso2.choreo.connect.enforcer.jmx;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.wso2.choreo.connect.enforcer.commons.logging.ErrorDetails;
+import org.wso2.choreo.connect.enforcer.commons.logging.LoggingConstants;
+
+import java.net.InetAddress;
+import java.rmi.registry.LocateRegistry;
+
+import javax.management.remote.JMXConnectorServer;
+import javax.management.remote.JMXConnectorServerFactory;
+import javax.management.remote.JMXServiceURL;
+
+/**
+ * JMX Connector Agent
+ */
+public class JMXAgent {
+    private static final Logger logger = LogManager.getLogger(JMXAgent.class);
+    private static JMXConnectorServer jmxConnectorServer;
+    private static final String DEFAULT_RMI_SERVER_PORT = "11111";
+    private static final String DEFAULT_RMI_REGISTRY_PORT = "9999";
+    private static final String JAVA_JMX_RMI_SERVICE_PORT = "com.sun.management.jmxremote.port";
+    private static final String JAVA_JMX_RMI_REGISTRY_PORT = "com.sun.management.jmxremote.rmi.port";
+
+    public static void initJMXAgent() {
+        if (JMXUtils.isJMXMetricsEnabled()) {
+            try {
+                String hostname = InetAddress.getLocalHost().getHostAddress();
+                String rmiServerPort = System.getProperty(JAVA_JMX_RMI_SERVICE_PORT, DEFAULT_RMI_SERVER_PORT);
+                String rmiRegistryPort = System.getProperty(JAVA_JMX_RMI_REGISTRY_PORT, DEFAULT_RMI_REGISTRY_PORT);
+
+                LocateRegistry.createRegistry(Integer.parseInt(rmiRegistryPort));
+
+                String jmxURL = String.format("service:jmx:rmi://%s:%s/jndi/rmi://%s:%s/jmxrmi", hostname,
+                        rmiServerPort,
+                        hostname, rmiRegistryPort);
+                JMXServiceURL jmxServiceURL = new JMXServiceURL(jmxURL);
+
+                jmxConnectorServer = JMXConnectorServerFactory.newJMXConnectorServer(jmxServiceURL, null,
+                        MBeanManagementFactory.getMBeanServer());
+                jmxConnectorServer.start();
+                logger.info("JMXAgent JMX Service URL : " + jmxServiceURL.toString());
+            } catch (Throwable throwable) {
+                logger.error("Failed to start JMX Agent", ErrorDetails.errorLog(LoggingConstants.Severity.MINOR, 6805),
+                        throwable);
+            }
+        }
+    }
+}

enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/JMXUtils.java

@@ -0,0 +1,34 @@
+/*
+ *  Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.wso2.choreo.connect.enforcer.jmx;
+
+/**
+ * JMX Utilities
+ */
+public class JMXUtils {
+
+    private static final String CHOREO_CONNECT_JMX_METRICS_ENABLE = "choreo.connect.jmx.metrics.enabled";
+
+    /**
+     * Returns true if jmx metrics enabled as a system property, otherwise false.
+     * 
+     * @return boolean
+     */
+    public static boolean isJMXMetricsEnabled() {
+        return Boolean.getBoolean(CHOREO_CONNECT_JMX_METRICS_ENABLE);
+    }
+}

enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/MBeanManagementFactory.java

@@ -0,0 +1,42 @@
+/*
+ *  Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.wso2.choreo.connect.enforcer.jmx;
+
+import javax.management.MBeanServer;
+import javax.management.MBeanServerFactory;
+
+/**
+ * The ManagementFactory class is a factory class for getting managed beans for
+ * the Enforcer.
+ */
+public class MBeanManagementFactory {
+
+    /*
+     * If one already exists, it will return that else it will create a new one and
+     * return.
+     * 
+     * @return A MBeanServer instance.
+     */
+    public static MBeanServer getMBeanServer() {
+        MBeanServer mBeanServer;
+        if (MBeanServerFactory.findMBeanServer(null).size() > 0) {
+            mBeanServer = MBeanServerFactory.findMBeanServer(null).get(0);
+        } else {
+            mBeanServer = MBeanServerFactory.createMBeanServer();
+        }
+        return mBeanServer;
+    }
+}

enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/MBeanRegistrator.java

@@ -0,0 +1,92 @@
+/*
+ *  Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.wso2.choreo.connect.enforcer.jmx;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.wso2.choreo.connect.enforcer.commons.logging.ErrorDetails;
+import org.wso2.choreo.connect.enforcer.commons.logging.LoggingConstants;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import javax.management.InstanceAlreadyExistsException;
+import javax.management.MBeanRegistrationException;
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.NotCompliantMBeanException;
+import javax.management.ObjectName;
+
+/**
+ * The class which is responsible for registering MBeans.
+ */
+public class MBeanRegistrator {
+    private static final Logger logger = LogManager.getLogger(MBeanRegistrator.class);
+    private static List<ObjectName> mBeans = new ArrayList<>();
+
+    private static final String SERVER_PACKAGE = "org.wso2.choreo.connect.enforcer";
+
+    private MBeanRegistrator() {
+    }
+
+    /**
+     * Registers an object as an MBean with the MBean server.
+     *
+     * @param mBeanInstance - The MBean to be registered as an MBean.
+     */
+    public static void registerMBean(Object mBeanInstance) throws RuntimeException {
+
+        if (JMXUtils.isJMXMetricsEnabled()) {
+            String className = mBeanInstance.getClass().getName();
+            if (className.indexOf('.') != -1) {
+                className = className.substring(className.lastIndexOf('.') + 1);
+            }
+
+            String objectName = String.format("%s:type=%s", SERVER_PACKAGE, className);
+            try {
+                MBeanServer mBeanServer = MBeanManagementFactory.getMBeanServer();
+                Set set = mBeanServer.queryNames(new ObjectName(objectName), null);
+                if (set.isEmpty()) {
+                    try {
+                        ObjectName name = new ObjectName(objectName);
+                        mBeanServer.registerMBean(mBeanInstance, name);
+                        mBeans.add(name);
+                    } catch (InstanceAlreadyExistsException e) {
+                        String msg = "MBean " + objectName + " already exists";
+                        logger.error(msg, ErrorDetails.errorLog(LoggingConstants.Severity.MINOR, 6801), e);
+                        throw new RuntimeException(msg, e);
+                    } catch (MBeanRegistrationException | NotCompliantMBeanException e) {
+                        String msg = "Execption when registering MBean";
+                        logger.error(msg, ErrorDetails.errorLog(LoggingConstants.Severity.MINOR, 6802), e);
+                        throw new RuntimeException(msg, e);
+                    }
+                } else {
+                    String msg = "MBean " + objectName + " already exists";
+                    logger.error(msg, ErrorDetails.errorLog(LoggingConstants.Severity.MINOR, 6803));
+                    throw new RuntimeException(msg);
+                }
+            } catch (MalformedObjectNameException e) {
+                String msg = "Could not register " + mBeanInstance.getClass() + " MBean";
+                logger.error(msg, ErrorDetails.errorLog(LoggingConstants.Severity.MINOR, 6804), e);
+                throw new RuntimeException(msg, e);
+            }
+        } else {
+            logger.debug("JMX Metrics should be enabled to register MBean instance: {}",
+                    mBeanInstance.getClass().getName());
+        }
+    }
+}

enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java

@@ -0,0 +1,62 @@
+/*
+ *  Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.wso2.choreo.connect.enforcer.metrics.jmx.api;
+
+/**
+ * MBean API for ExtAuth Service metrics.
+ */
+public interface ExtAuthMetricsMXBean {
+
+    /**
+     * Getter for total request count.
+     * 
+     * @return long
+     */
+    public long getTotalRequestCount();
+
+    /**
+     * Getter for average response time in milli seconds.
+     * 
+     * @return long
+     */
+    public long getAverageResponseTimeMillis();
+
+    /**
+     * Getter for maximum response time in milliseconds.
+     * 
+     * @return long
+     */
+    public long getMaxResponseTimeMillis();
+
+    /**
+     * Getter for mimnimum response time in milliseconds.
+     * 
+     * @return long
+     */
+    public long getMinResponseTimeMillis();
+
+    /**
+     * Resets all the metrics to thier initial values.
+     */
+    public void resetExtAuthMetrics();
+
+    /**
+     * Resets all the metrics to thier initial values.
+     */
+    public long getRequestCountInLastFiveMinutes();
+
+}