diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/ConfigHolder.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/ConfigHolder.java index 444ecc1506..e2e306fa2f 100644 --- a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/ConfigHolder.java +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/ConfigHolder.java @@ -68,6 +68,7 @@ import org.wso2.choreo.connect.enforcer.config.dto.TracingDTO; import org.wso2.choreo.connect.enforcer.constants.APIConstants; import org.wso2.choreo.connect.enforcer.constants.Constants; +import org.wso2.choreo.connect.enforcer.jmx.MBeanRegistrator; import org.wso2.choreo.connect.enforcer.throttle.databridge.agent.conf.AgentConfiguration; import org.wso2.choreo.connect.enforcer.util.BackendJwtUtils; import org.wso2.choreo.connect.enforcer.util.FilterUtils; @@ -235,6 +236,8 @@ private void populateAuthService(Service cdsAuth) { authDto.setMaxMessageSize(cdsAuth.getMaxMessageSize()); ThreadPoolConfig threadPool = new ThreadPoolConfig(); + MBeanRegistrator.registerMBean(threadPool); + threadPool.setCoreSize(cdsAuth.getThreadPool().getCoreSize()); threadPool.setKeepAliveTime(cdsAuth.getThreadPool().getKeepAliveTime()); threadPool.setMaxSize(cdsAuth.getThreadPool().getMaxSize()); diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/dto/ThreadPoolConfig.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/dto/ThreadPoolConfig.java index 31a1443f3f..cecbb3fc8b 100644 --- a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/dto/ThreadPoolConfig.java +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/dto/ThreadPoolConfig.java @@ -21,7 +21,7 @@ /** * Holds the configurations related to threading of gRPC netty server. */ -public class ThreadPoolConfig { +public class ThreadPoolConfig implements ThreadPoolConfigMBean { private int coreSize; private int maxSize; private int keepAliveTime; diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/dto/ThreadPoolConfigMBean.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/dto/ThreadPoolConfigMBean.java new file mode 100644 index 0000000000..167cbcce61 --- /dev/null +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/config/dto/ThreadPoolConfigMBean.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.wso2.choreo.connect.enforcer.config.dto; + +/** + * MBean API for Thread Pool Configuration. + */ +public interface ThreadPoolConfigMBean { + /** + * Getter for core size. + * + * @return int + */ + public int getCoreSize(); + + /** + * Getter for max size. + * + * @return int + */ + public int getMaxSize(); + + /** + * Getter for keep alive size. + * + * @return int + */ + public int getKeepAliveTime(); + + /** + * Getter for queue size. + * + * @return int + */ + public int getQueueSize(); +} diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/grpc/ExtAuthService.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/grpc/ExtAuthService.java index 792524c8ca..32cef31433 100644 --- a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/grpc/ExtAuthService.java +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/grpc/ExtAuthService.java @@ -39,8 +39,10 @@ import org.wso2.choreo.connect.enforcer.constants.HttpConstants; import org.wso2.choreo.connect.enforcer.constants.RouterAccessLogConstants; import org.wso2.choreo.connect.enforcer.deniedresponse.DeniedResponsePreparer; +import org.wso2.choreo.connect.enforcer.jmx.JMXUtils; import org.wso2.choreo.connect.enforcer.metrics.MetricsExporter; import org.wso2.choreo.connect.enforcer.metrics.MetricsManager; +import org.wso2.choreo.connect.enforcer.metrics.jmx.impl.ExtAuthMetrics; import org.wso2.choreo.connect.enforcer.server.HttpRequestHandler; import org.wso2.choreo.connect.enforcer.tracing.TracingConstants; import org.wso2.choreo.connect.enforcer.tracing.TracingContextHolder; @@ -93,6 +95,9 @@ public void check(CheckRequest request, StreamObserver responseOb MetricsExporter metricsExporter = MetricsManager.getInstance(); metricsExporter.trackMetric("enforcerLatency", System.currentTimeMillis() - starTimestamp); } + if (JMXUtils.isJMXMetricsEnabled()) { + ExtAuthMetrics.getInstance().recordMetric(System.currentTimeMillis() - starTimestamp); + } } } diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/JMXAgent.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/JMXAgent.java new file mode 100644 index 0000000000..6a2658437a --- /dev/null +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/JMXAgent.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.wso2.choreo.connect.enforcer.jmx; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.wso2.choreo.connect.enforcer.commons.logging.ErrorDetails; +import org.wso2.choreo.connect.enforcer.commons.logging.LoggingConstants; + +import java.net.InetAddress; +import java.rmi.registry.LocateRegistry; + +import javax.management.remote.JMXConnectorServer; +import javax.management.remote.JMXConnectorServerFactory; +import javax.management.remote.JMXServiceURL; + +/** + * JMX Connector Agent + */ +public class JMXAgent { + private static final Logger logger = LogManager.getLogger(JMXAgent.class); + private static JMXConnectorServer jmxConnectorServer; + private static final String DEFAULT_RMI_SERVER_PORT = "11111"; + private static final String DEFAULT_RMI_REGISTRY_PORT = "9999"; + private static final String JAVA_JMX_RMI_SERVICE_PORT = "com.sun.management.jmxremote.port"; + private static final String JAVA_JMX_RMI_REGISTRY_PORT = "com.sun.management.jmxremote.rmi.port"; + + public static void initJMXAgent() { + if (JMXUtils.isJMXMetricsEnabled()) { + try { + String hostname = InetAddress.getLocalHost().getHostAddress(); + String rmiServerPort = System.getProperty(JAVA_JMX_RMI_SERVICE_PORT, DEFAULT_RMI_SERVER_PORT); + String rmiRegistryPort = System.getProperty(JAVA_JMX_RMI_REGISTRY_PORT, DEFAULT_RMI_REGISTRY_PORT); + + LocateRegistry.createRegistry(Integer.parseInt(rmiRegistryPort)); + + String jmxURL = String.format("service:jmx:rmi://%s:%s/jndi/rmi://%s:%s/jmxrmi", hostname, + rmiServerPort, + hostname, rmiRegistryPort); + JMXServiceURL jmxServiceURL = new JMXServiceURL(jmxURL); + + jmxConnectorServer = JMXConnectorServerFactory.newJMXConnectorServer(jmxServiceURL, null, + MBeanManagementFactory.getMBeanServer()); + jmxConnectorServer.start(); + logger.info("JMXAgent JMX Service URL : " + jmxServiceURL.toString()); + } catch (Throwable throwable) { + logger.error("Failed to start JMX Agent", ErrorDetails.errorLog(LoggingConstants.Severity.MINOR, 6805), + throwable); + } + } + } +} diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/JMXUtils.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/JMXUtils.java new file mode 100644 index 0000000000..438e6f521f --- /dev/null +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/JMXUtils.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.wso2.choreo.connect.enforcer.jmx; + +/** + * JMX Utilities + */ +public class JMXUtils { + + private static final String CHOREO_CONNECT_JMX_METRICS_ENABLE = "choreo.connect.jmx.metrics.enabled"; + + /** + * Returns true if jmx metrics enabled as a system property, otherwise false. + * + * @return boolean + */ + public static boolean isJMXMetricsEnabled() { + return Boolean.getBoolean(CHOREO_CONNECT_JMX_METRICS_ENABLE); + } +} diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/MBeanManagementFactory.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/MBeanManagementFactory.java new file mode 100644 index 0000000000..c46b003351 --- /dev/null +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/MBeanManagementFactory.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.wso2.choreo.connect.enforcer.jmx; + +import javax.management.MBeanServer; +import javax.management.MBeanServerFactory; + +/** + * The ManagementFactory class is a factory class for getting managed beans for + * the Enforcer. + */ +public class MBeanManagementFactory { + + /* + * If one already exists, it will return that else it will create a new one and + * return. + * + * @return A MBeanServer instance. + */ + public static MBeanServer getMBeanServer() { + MBeanServer mBeanServer; + if (MBeanServerFactory.findMBeanServer(null).size() > 0) { + mBeanServer = MBeanServerFactory.findMBeanServer(null).get(0); + } else { + mBeanServer = MBeanServerFactory.createMBeanServer(); + } + return mBeanServer; + } +} diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/MBeanRegistrator.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/MBeanRegistrator.java new file mode 100644 index 0000000000..ad208ba274 --- /dev/null +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/jmx/MBeanRegistrator.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.wso2.choreo.connect.enforcer.jmx; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.wso2.choreo.connect.enforcer.commons.logging.ErrorDetails; +import org.wso2.choreo.connect.enforcer.commons.logging.LoggingConstants; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import javax.management.InstanceAlreadyExistsException; +import javax.management.MBeanRegistrationException; +import javax.management.MBeanServer; +import javax.management.MalformedObjectNameException; +import javax.management.NotCompliantMBeanException; +import javax.management.ObjectName; + +/** + * The class which is responsible for registering MBeans. + */ +public class MBeanRegistrator { + private static final Logger logger = LogManager.getLogger(MBeanRegistrator.class); + private static List mBeans = new ArrayList<>(); + + private static final String SERVER_PACKAGE = "org.wso2.choreo.connect.enforcer"; + + private MBeanRegistrator() { + } + + /** + * Registers an object as an MBean with the MBean server. + * + * @param mBeanInstance - The MBean to be registered as an MBean. + */ + public static void registerMBean(Object mBeanInstance) throws RuntimeException { + + if (JMXUtils.isJMXMetricsEnabled()) { + String className = mBeanInstance.getClass().getName(); + if (className.indexOf('.') != -1) { + className = className.substring(className.lastIndexOf('.') + 1); + } + + String objectName = String.format("%s:type=%s", SERVER_PACKAGE, className); + try { + MBeanServer mBeanServer = MBeanManagementFactory.getMBeanServer(); + Set set = mBeanServer.queryNames(new ObjectName(objectName), null); + if (set.isEmpty()) { + try { + ObjectName name = new ObjectName(objectName); + mBeanServer.registerMBean(mBeanInstance, name); + mBeans.add(name); + } catch (InstanceAlreadyExistsException e) { + String msg = "MBean " + objectName + " already exists"; + logger.error(msg, ErrorDetails.errorLog(LoggingConstants.Severity.MINOR, 6801), e); + throw new RuntimeException(msg, e); + } catch (MBeanRegistrationException | NotCompliantMBeanException e) { + String msg = "Execption when registering MBean"; + logger.error(msg, ErrorDetails.errorLog(LoggingConstants.Severity.MINOR, 6802), e); + throw new RuntimeException(msg, e); + } + } else { + String msg = "MBean " + objectName + " already exists"; + logger.error(msg, ErrorDetails.errorLog(LoggingConstants.Severity.MINOR, 6803)); + throw new RuntimeException(msg); + } + } catch (MalformedObjectNameException e) { + String msg = "Could not register " + mBeanInstance.getClass() + " MBean"; + logger.error(msg, ErrorDetails.errorLog(LoggingConstants.Severity.MINOR, 6804), e); + throw new RuntimeException(msg, e); + } + } else { + logger.debug("JMX Metrics should be enabled to register MBean instance: {}", + mBeanInstance.getClass().getName()); + } + } +} diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java new file mode 100644 index 0000000000..cfa5c15778 --- /dev/null +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.wso2.choreo.connect.enforcer.metrics.jmx.api; + +/** + * MBean API for ExtAuth Service metrics. + */ +public interface ExtAuthMetricsMXBean { + + /** + * Getter for total request count. + * + * @return long + */ + public long getTotalRequestCount(); + + /** + * Getter for average response time in milli seconds. + * + * @return long + */ + public long getAverageResponseTimeMillis(); + + /** + * Getter for maximum response time in milliseconds. + * + * @return long + */ + public long getMaxResponseTimeMillis(); + + /** + * Getter for mimnimum response time in milliseconds. + * + * @return long + */ + public long getMinResponseTimeMillis(); + + /** + * Resets all the metrics to thier initial values. + */ + public void resetExtAuthMetrics(); + + /** + * Resets all the metrics to thier initial values. + */ + public long getRequestCountInLastFiveMinutes(); + +} diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/impl/ExtAuthMetrics.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/impl/ExtAuthMetrics.java new file mode 100644 index 0000000000..7e975e6d4b --- /dev/null +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/impl/ExtAuthMetrics.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.wso2.choreo.connect.enforcer.metrics.jmx.impl; + +import org.wso2.choreo.connect.enforcer.jmx.MBeanRegistrator; +import org.wso2.choreo.connect.enforcer.metrics.jmx.api.ExtAuthMetricsMXBean; + +import java.util.Timer; +import java.util.TimerTask; + +/** + * Singleton MBean for ExtAuth Service metrics. + */ +public class ExtAuthMetrics extends TimerTask implements ExtAuthMetricsMXBean { + + private static final long REQUEST_COUNT_INTERVAL_MILLIS = 5 * 60 * 1000; + private static ExtAuthMetrics extAuthMetricsMBean = null; + + private long requestCountInLastFiveMinutes = 0; + private long totalRequestCount = 0; + private long averageResponseTimeMillis = 0; + private long maxResponseTimeMillis = Long.MIN_VALUE; + private long minResponseTimeMillis = Long.MAX_VALUE; + + private ExtAuthMetrics() { + MBeanRegistrator.registerMBean(this); + } + + /** + * Getter for the Singleton ExtAuthMetrics instance. + * + * @return ExtAuthMetrics + */ + public static ExtAuthMetrics getInstance() { + if (extAuthMetricsMBean == null) { + synchronized (ExtAuthMetrics.class) { + if (extAuthMetricsMBean == null) { + Timer timer = new Timer(); + extAuthMetricsMBean = new ExtAuthMetrics(); + timer.schedule(extAuthMetricsMBean, 0, REQUEST_COUNT_INTERVAL_MILLIS); + } + } + } + return extAuthMetricsMBean; + } + + @Override + public long getTotalRequestCount() { + return totalRequestCount; + }; + + @Override + public long getAverageResponseTimeMillis() { + return averageResponseTimeMillis; + }; + + @Override + public long getMaxResponseTimeMillis() { + return maxResponseTimeMillis; + }; + + @Override + public long getMinResponseTimeMillis() { + return minResponseTimeMillis; + }; + + public synchronized void recordMetric(long responseTimeMillis) { + this.requestCountInLastFiveMinutes += 1; + this.totalRequestCount += 1; + this.averageResponseTimeMillis = (this.averageResponseTimeMillis + responseTimeMillis) / totalRequestCount; + this.minResponseTimeMillis = Math.min(this.minResponseTimeMillis, responseTimeMillis); + this.maxResponseTimeMillis = Math.max(this.maxResponseTimeMillis, responseTimeMillis); + } + + @Override + public synchronized void resetExtAuthMetrics() { + this.totalRequestCount = 0; + this.averageResponseTimeMillis = 0; + this.maxResponseTimeMillis = Long.MIN_VALUE; + this.minResponseTimeMillis = Long.MAX_VALUE; + } + + @Override + public synchronized void run() { + requestCountInLastFiveMinutes = 0; + } + + @Override + public long getRequestCountInLastFiveMinutes() { + return requestCountInLastFiveMinutes; + } +} diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/server/AuthServer.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/server/AuthServer.java index 37b8c388f2..5a6d6b8cbd 100644 --- a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/server/AuthServer.java +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/server/AuthServer.java @@ -43,6 +43,7 @@ import org.wso2.choreo.connect.enforcer.grpc.WebSocketFrameService; import org.wso2.choreo.connect.enforcer.grpc.interceptors.AccessLogInterceptor; import org.wso2.choreo.connect.enforcer.grpc.interceptors.OpenTelemetryInterceptor; +import org.wso2.choreo.connect.enforcer.jmx.JMXAgent; import org.wso2.choreo.connect.enforcer.keymgt.KeyManagerHolder; import org.wso2.choreo.connect.enforcer.metrics.MetricsManager; import org.wso2.choreo.connect.enforcer.security.jwt.validator.RevokedJWTDataHolder; @@ -137,6 +138,9 @@ public static void main(String[] args) { server.start(); logger.info("Sever started Listening in port : " + 8081); + // Initialize JMX Agent + JMXAgent.initJMXAgent(); + //TODO: Get the tenant domain from config SubscriptionDataHolder.getInstance().getTenantSubscriptionStore().initializeStore(); KeyManagerHolder.getInstance().init();