[pinpoint-apm#9666] Realtime activeThreadCount with Redis

smilu97 · Feb 15, 2023 · 2beb953 · 2beb953
1 parent 3702da1
commit 2beb953
Show file tree

Hide file tree

Showing 92 changed files with 4,219 additions and 44 deletions.
diff --git a/collector/src/main/java/com/navercorp/pinpoint/collector/CollectorAppPropertySources.java b/collector/src/main/java/com/navercorp/pinpoint/collector/CollectorAppPropertySources.java
@@ -25,6 +25,7 @@
         @PropertySource(name = "CollectorAppPropertySources-GRPC", value = { CollectorAppPropertySources.GRPC_ROOT, CollectorAppPropertySources.GRPC_PROFILE}),
         @PropertySource(name = "CollectorAppPropertySources-HBase", value = { CollectorAppPropertySources.HBASE_ROOT, CollectorAppPropertySources.HBASE_PROFILE}),
         @PropertySource(name = "CollectorAppPropertySources-JDBC", value = { CollectorAppPropertySources.JDBC_ROOT, CollectorAppPropertySources.JDBC_PROFILE}),
+        @PropertySource(name = "WebAppPropertySources-Redis", value = { CollectorAppPropertySources.REDIS_ROOT, CollectorAppPropertySources.REDIS_PROFILE}),
 })
 public final class CollectorAppPropertySources {
     public static final String HBASE_ROOT= "classpath:hbase-root.properties";
@@ -36,6 +37,9 @@ public final class CollectorAppPropertySources {
     public static final String JDBC_ROOT = "classpath:jdbc-root.properties";
     public static final String JDBC_PROFILE = "classpath:profiles/${pinpoint.profiles.active:local}/jdbc.properties";
 
+    public static final String REDIS_ROOT = "classpath:redis-root.properties";
+    public static final String REDIS_PROFILE = "classpath:profiles/${pinpoint.profiles.active:release}/redis.properties";
+
     public static final String COLLECTOR_ROOT = "classpath:pinpoint-collector-root.properties";
     public static final String COLLECTOR_PROFILE = "classpath:profiles/${pinpoint.profiles.active:local}/pinpoint-collector.properties";
 

diff --git a/...or/src/main/java/com/navercorp/pinpoint/collector/cluster/route/AbstractRouteHandler.java b/...or/src/main/java/com/navercorp/pinpoint/collector/cluster/route/AbstractRouteHandler.java
@@ -22,9 +22,8 @@
 import com.navercorp.pinpoint.thrift.dto.command.TCommandTransfer;
 import com.navercorp.pinpoint.thrift.dto.command.TCommandTransferResponse;
 import com.navercorp.pinpoint.thrift.dto.command.TRouteResult;
-
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -47,7 +46,10 @@ protected ClusterPoint<?> findClusterPoint(TCommandTransfer deliveryCommand) {
         String agentId = deliveryCommand.getAgentId();
         long startTimeStamp = deliveryCommand.getStartTime();
         final ClusterKey sourceKey = new ClusterKey(applicationName, agentId, startTimeStamp);
+        return findClusterPoint(sourceKey);
+    }
 
+    public ClusterPoint<?> findClusterPoint(ClusterKey sourceKey) {
         List<ClusterPoint<?>> result = new ArrayList<>();
 
         for (ClusterPoint<?> targetClusterPoint : targetClusterPointLocator.getClusterPointList()) {
@@ -62,7 +64,7 @@ protected ClusterPoint<?> findClusterPoint(TCommandTransfer deliveryCommand) {
         }
 
         if (result.size() > 1) {
-            logger.warn("Ambiguous ClusterPoint {}, {}, {} (Valid Agent list={}).", applicationName, agentId, startTimeStamp, result);
+            logger.warn("Ambiguous ClusterPoint {} (Valid Agent list={}).", sourceKey, result);
             return null;
         }
 

diff --git a/collector/src/main/resources/pinpoint-collector-root.properties b/collector/src/main/resources/pinpoint-collector-root.properties
@@ -147,6 +147,11 @@ flink.cluster.zookeeper.address=${pinpoint.zookeeper.address}
 flink.cluster.zookeeper.znode_root=/pinpoint-cluster
 flink.cluster.zookeeper.sessiontimeout=3000
 
+# Active Thread Count
+pinpoint.collector.realtime.atc.demand.duration=12500
+pinpoint.collector.realtime.atc.supply.throttle.termMillis=100
+pinpoint.collector.realtime.atc.enable-count-metric=false
+
 ###########################################################
 # BANNER                                                  #
 ###########################################################
@@ -199,4 +204,7 @@ pinpoint.banner.configs=spring.active.profile,\
                         hbase.client.host,\
                         hbase.client.port,\
                         hbase.zookeeper.znode.parent,\
-                        hbase.namespace
+                        hbase.namespace,\
+                        spring.data.redis.host,\
+                        spring.data.redis.port,\
+                        spring.data.redis.cluster.nodes
diff --git a/collector/src/main/resources/profiles/local/redis.properties b/collector/src/main/resources/profiles/local/redis.properties
@@ -0,0 +1,13 @@
+spring.data.redis.lettuce.client.io-thread-pool-size=8
+spring.data.redis.lettuce.client.computation-thread-pool-size=8
+spring.data.redis.lettuce.client.request-queue-size=1024
+
+spring.data.redis.username=default
+spring.data.redis.password=
+
+# Standalone mode
+spring.data.redis.host=localhost
+spring.data.redis.port=6379
+
+# Cluster mode: Cluster mode is prior than Standalone
+spring.data.redis.cluster.nodes=
diff --git a/collector/src/main/resources/profiles/release/redis.properties b/collector/src/main/resources/profiles/release/redis.properties
@@ -0,0 +1,13 @@
+spring.data.redis.lettuce.client.io-thread-pool-size=8
+spring.data.redis.lettuce.client.computation-thread-pool-size=8
+spring.data.redis.lettuce.client.request-queue-size=1024
+
+spring.data.redis.username=default
+spring.data.redis.password=
+
+# Standalone mode
+spring.data.redis.host=localhost
+spring.data.redis.port=6379
+
+# Cluster mode: Cluster mode is prior than Standalone
+spring.data.redis.cluster.nodes=
diff --git a/collector/src/main/resources/redis-root.properties b/collector/src/main/resources/redis-root.properties
@@ -0,0 +1 @@
+spring.data.redis.lettuce.client.name=collectorRedis
diff --git a/metric-module/collector-starter/pom.xml b/metric-module/collector-starter/pom.xml
@@ -34,6 +34,10 @@
             <groupId>com.navercorp.pinpoint</groupId>
             <artifactId>pinpoint-uristat-collector</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.navercorp.pinpoint</groupId>
+            <artifactId>pinpoint-realtime-collector</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/...in/java/com/navercorp/pinpoint/collector/starter/multi/application/BasicCollectorApp.java b/...in/java/com/navercorp/pinpoint/collector/starter/multi/application/BasicCollectorApp.java
@@ -4,6 +4,10 @@
 import com.navercorp.pinpoint.collector.config.FlinkContextConfiguration;
 import org.springframework.boot.SpringBootConfiguration;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisRepositoriesAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.autoconfigure.sql.init.SqlInitializationAutoConfiguration;
 import org.springframework.boot.autoconfigure.transaction.TransactionAutoConfiguration;
@@ -12,7 +16,15 @@
 import org.springframework.context.annotation.ImportResource;
 
 @SpringBootConfiguration
-@EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class, TransactionAutoConfiguration.class, SqlInitializationAutoConfiguration.class})
+@EnableAutoConfiguration(exclude = {
+        DataSourceAutoConfiguration.class,
+        TransactionAutoConfiguration.class,
+        SqlInitializationAutoConfiguration.class,
+        SpringDataWebAutoConfiguration.class,
+        RedisAutoConfiguration.class,
+        RedisRepositoriesAutoConfiguration.class,
+        RedisReactiveAutoConfiguration.class
+})
 @ImportResource({"classpath:applicationContext-collector.xml", "classpath:servlet-context-collector.xml"})
 @Import({CollectorAppPropertySources.class, FlinkContextConfiguration.class})
 @ComponentScan({})

diff --git a/...ain/java/com/navercorp/pinpoint/collector/starter/multi/application/MultiApplication.java b/...ain/java/com/navercorp/pinpoint/collector/starter/multi/application/MultiApplication.java
@@ -10,19 +10,31 @@
 import com.navercorp.pinpoint.metric.collector.CollectorTypeParser;
 import com.navercorp.pinpoint.metric.collector.MetricCollectorApp;
 import com.navercorp.pinpoint.metric.collector.TypeSet;
+import com.navercorp.pinpoint.realtime.collector.atc.config.RealtimeCollectorConfig;
 import com.navercorp.pinpoint.uristat.collector.UriStatCollectorConfig;
 import org.springframework.boot.Banner;
 import org.springframework.boot.SpringBootConfiguration;
 import org.springframework.boot.WebApplicationType;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisRepositoriesAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.autoconfigure.transaction.TransactionAutoConfiguration;
 import org.springframework.boot.builder.SpringApplicationBuilder;
 
 import java.util.Arrays;
 
 @SpringBootConfiguration
-@EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class, TransactionAutoConfiguration.class})
+@EnableAutoConfiguration(exclude = {
+        DataSourceAutoConfiguration.class,
+        TransactionAutoConfiguration.class,
+        SpringDataWebAutoConfiguration.class,
+        RedisAutoConfiguration.class,
+        RedisRepositoriesAutoConfiguration.class,
+        RedisReactiveAutoConfiguration.class
+})
 public class MultiApplication {
     private static final ServerBootLogger logger = ServerBootLogger.getLogger(MultiApplication.class);
 
@@ -44,7 +56,11 @@ public static void main(String[] args) {
 
         if (types.hasType(CollectorType.BASIC)) {
             logger.info(String.format("Start %s collector", CollectorType.BASIC));
-            SpringApplicationBuilder collectorAppBuilder = createAppBuilder(builder, 15400, BasicCollectorApp.class, UriStatCollectorConfig.class);
+            SpringApplicationBuilder collectorAppBuilder = createAppBuilder(builder, 15400,
+                    BasicCollectorApp.class,
+                    UriStatCollectorConfig.class,
+                    RealtimeCollectorConfig.class
+            );
             collectorAppBuilder.build().run(args);
         }
 

diff --git a/metric-module/web-starter/pom.xml b/metric-module/web-starter/pom.xml
@@ -44,6 +44,10 @@
             <groupId>com.navercorp.pinpoint</groupId>
             <artifactId>pinpoint-uristat-web</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.navercorp.pinpoint</groupId>
+            <artifactId>pinpoint-realtime-web</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/...e/web-starter/src/main/java/com/navercorp/pinpoint/web/starter/multi/MetricAndWebApp.java b/...e/web-starter/src/main/java/com/navercorp/pinpoint/web/starter/multi/MetricAndWebApp.java
@@ -18,17 +18,22 @@
 
 import com.navercorp.pinpoint.common.server.util.ServerBootLogger;
 import com.navercorp.pinpoint.metric.web.MetricWebApp;
+import com.navercorp.pinpoint.realtime.web.atc.config.RealtimeWebConfig;
 import com.navercorp.pinpoint.uristat.web.UriStatWebConfig;
+import com.navercorp.pinpoint.web.AuthorizationConfig;
 import com.navercorp.pinpoint.web.PinpointBasicLoginConfig;
 import com.navercorp.pinpoint.web.WebApp;
 import com.navercorp.pinpoint.web.WebAppPropertySources;
 import com.navercorp.pinpoint.web.WebMvcConfig;
 import com.navercorp.pinpoint.web.WebServerConfig;
 import com.navercorp.pinpoint.web.WebStarter;
-import com.navercorp.pinpoint.web.AuthorizationConfig;
 import com.navercorp.pinpoint.web.cache.CacheConfiguration;
 import org.springframework.boot.SpringBootConfiguration;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisRepositoriesAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.transaction.TransactionAutoConfiguration;
@@ -38,16 +43,30 @@
  * @author minwoo.jung
  */
 @SpringBootConfiguration
-@EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class, TransactionAutoConfiguration.class,
-        SecurityAutoConfiguration.class})
+@EnableAutoConfiguration(exclude = {
+        DataSourceAutoConfiguration.class,
+        TransactionAutoConfiguration.class,
+        SecurityAutoConfiguration.class,
+        SpringDataWebAutoConfiguration.class,
+        RedisAutoConfiguration.class,
+        RedisRepositoriesAutoConfiguration.class,
+        RedisReactiveAutoConfiguration.class
+})
 @ImportResource({"classpath:applicationContext-web.xml", "classpath:servlet-context-web.xml"})
 @Import({WebAppPropertySources.class, WebServerConfig.class, WebMvcConfig.class, CacheConfiguration.class})
 public class MetricAndWebApp {
     private static final ServerBootLogger logger = ServerBootLogger.getLogger(WebApp.class);
 
     public static void main(String[] args) {
         try {
-            WebStarter starter = new WebStarter(MetricAndWebApp.class, PinpointBasicLoginConfig.class, AuthorizationConfig.class, MetricWebApp.class, UriStatWebConfig.class);
+            WebStarter starter = new WebStarter(
+                    MetricAndWebApp.class,
+                    PinpointBasicLoginConfig.class,
+                    AuthorizationConfig.class,
+                    MetricWebApp.class,
+                    UriStatWebConfig.class,
+                    RealtimeWebConfig.class
+            );
             starter.start(args);
         } catch (Exception exception) {
             logger.error("[WebApp] could not launch app.", exception);

diff --git a/pom.xml b/pom.xml
@@ -119,6 +119,7 @@
         <module>hbase2-module</module>
         <module>metric-module</module>
         <module>uristat</module>
+        <module>realtime</module>
 
         <!-- <module>agent-testweb</module> -->
         <!-- <module>plugins-it</module> -->
@@ -439,6 +440,21 @@
                 <artifactId>pinpoint-uristat-collector</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>com.navercorp.pinpoint</groupId>
+                <artifactId>pinpoint-realtime-web</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.navercorp.pinpoint</groupId>
+                <artifactId>pinpoint-realtime-collector</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.navercorp.pinpoint</groupId>
+                <artifactId>pinpoint-realtime-common</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>com.navercorp.pinpoint</groupId>
                 <artifactId>pinpoint-metric</artifactId>

diff --git a/realtime/pom.xml b/realtime/pom.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>pinpoint</artifactId>
+        <groupId>com.navercorp.pinpoint</groupId>
+        <version>2.5.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>pinpoint-realtime</artifactId>
+    <packaging>pom</packaging>
+
+    <modules>
+        <module>realtime-web</module>
+        <module>realtime-common</module>
+        <module>realtime-collector</module>
+    </modules>
+
+</project>
diff --git a/realtime/realtime-collector/pom.xml b/realtime/realtime-collector/pom.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>pinpoint-realtime</artifactId>
+        <groupId>com.navercorp.pinpoint</groupId>
+        <version>2.5.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>pinpoint-realtime-collector</artifactId>
+
+    <properties>
+        <jdk.version>11</jdk.version>
+        <jdk.home>${env.JAVA_11_HOME}</jdk.home>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.navercorp.pinpoint</groupId>
+            <artifactId>pinpoint-realtime-common</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.navercorp.pinpoint</groupId>
+            <artifactId>pinpoint-collector</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>