[pinpoint-apm#9666] Add redis pubsub atc,atd,echo

smilu97 · Apr 17, 2023 · 9cef2af · 9cef2af
1 parent 2fff373
commit 9cef2af
Show file tree

Hide file tree

Showing 143 changed files with 7,607 additions and 414 deletions.
diff --git a/collector/src/main/java/com/navercorp/pinpoint/collector/CollectorAppPropertySources.java b/collector/src/main/java/com/navercorp/pinpoint/collector/CollectorAppPropertySources.java
@@ -24,6 +24,7 @@
         @PropertySource(name = "CollectorAppPropertySources", value = { CollectorAppPropertySources.COLLECTOR_ROOT, CollectorAppPropertySources.COLLECTOR_PROFILE}),
         @PropertySource(name = "CollectorAppPropertySources-GRPC", value = { CollectorAppPropertySources.GRPC_ROOT, CollectorAppPropertySources.GRPC_PROFILE}),
         @PropertySource(name = "CollectorAppPropertySources-JDBC", value = { CollectorAppPropertySources.JDBC_ROOT, CollectorAppPropertySources.JDBC_PROFILE}),
+        @PropertySource(name = "CollectorAppPropertySources-Redis", value = { CollectorAppPropertySources.REDIS_ROOT, CollectorAppPropertySources.REDIS_PROFILE}),
 })
 public final class CollectorAppPropertySources {
     public static final String GRPC_ROOT = "classpath:pinpoint-collector-grpc-root.properties";
@@ -32,6 +33,9 @@ public final class CollectorAppPropertySources {
     public static final String JDBC_ROOT = "classpath:jdbc-root.properties";
     public static final String JDBC_PROFILE = "classpath:profiles/${pinpoint.profiles.active:local}/jdbc.properties";
 
+    public static final String REDIS_ROOT = "classpath:redis-root.properties";
+    public static final String REDIS_PROFILE = "classpath:profiles/${pinpoint.profiles.active:local}/redis.properties";
+
     public static final String COLLECTOR_ROOT = "classpath:pinpoint-collector-root.properties";
     public static final String COLLECTOR_PROFILE = "classpath:profiles/${pinpoint.profiles.active:local}/pinpoint-collector.properties";
 

diff --git a/...or/src/main/java/com/navercorp/pinpoint/collector/cluster/route/AbstractRouteHandler.java b/...or/src/main/java/com/navercorp/pinpoint/collector/cluster/route/AbstractRouteHandler.java
@@ -22,9 +22,8 @@
 import com.navercorp.pinpoint.thrift.dto.command.TCommandTransfer;
 import com.navercorp.pinpoint.thrift.dto.command.TCommandTransferResponse;
 import com.navercorp.pinpoint.thrift.dto.command.TRouteResult;
-
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -47,7 +46,10 @@ protected ClusterPoint<?> findClusterPoint(TCommandTransfer deliveryCommand) {
         String agentId = deliveryCommand.getAgentId();
         long startTimeStamp = deliveryCommand.getStartTime();
         final ClusterKey sourceKey = new ClusterKey(applicationName, agentId, startTimeStamp);
+        return findClusterPoint(sourceKey);
+    }
 
+    public ClusterPoint<?> findClusterPoint(ClusterKey sourceKey) {
         List<ClusterPoint<?>> result = new ArrayList<>();
 
         for (ClusterPoint<?> targetClusterPoint : targetClusterPointLocator.getClusterPointList()) {
@@ -62,7 +64,7 @@ protected ClusterPoint<?> findClusterPoint(TCommandTransfer deliveryCommand) {
         }
 
         if (result.size() > 1) {
-            logger.warn("Ambiguous ClusterPoint {}, {}, {} (Valid Agent list={}).", applicationName, agentId, startTimeStamp, result);
+            logger.warn("Ambiguous ClusterPoint {} (Valid Agent list={}).", sourceKey, result);
             return null;
         }
 

diff --git a/...src/main/java/com/navercorp/pinpoint/collector/grpc/config/GrpcReceiverConfiguration.java b/...src/main/java/com/navercorp/pinpoint/collector/grpc/config/GrpcReceiverConfiguration.java
@@ -46,7 +46,7 @@ public class GrpcReceiverConfiguration {
     private final ServerOption serverOption;
 
 
-    GrpcReceiverConfiguration(boolean enable,
+    protected GrpcReceiverConfiguration(boolean enable,
                                      BindAddress bindAddress,
                                      ExecutorConfiguration serverExecutor,
                                      ExecutorConfiguration serverCallExecutor,

diff --git a/collector/src/main/resources/pinpoint-collector-root.properties b/collector/src/main/resources/pinpoint-collector-root.properties
@@ -147,6 +147,11 @@ flink.cluster.zookeeper.address=${pinpoint.zookeeper.address}
 flink.cluster.zookeeper.znode_root=/pinpoint-cluster
 flink.cluster.zookeeper.sessiontimeout=3000
 
+# Active Thread Count
+pinpoint.collector.realtime.atc.demand.duration=12500
+pinpoint.collector.realtime.atc.supply.throttle.termMillis=100
+pinpoint.collector.realtime.atc.enable-count-metric=false
+
 ###########################################################
 # BANNER                                                  #
 ###########################################################
@@ -199,4 +204,7 @@ pinpoint.banner.configs=spring.active.profile,\
                         hbase.client.host,\
                         hbase.client.port,\
                         hbase.zookeeper.znode.parent,\
-                        hbase.namespace
+                        hbase.namespace,\
+                        spring.data.redis.host,\
+                        spring.data.redis.port,\
+                        spring.data.redis.cluster.nodes
diff --git a/collector/src/main/resources/profiles/local/redis.properties b/collector/src/main/resources/profiles/local/redis.properties
@@ -0,0 +1,13 @@
+spring.data.redis.lettuce.client.io-thread-pool-size=8
+spring.data.redis.lettuce.client.computation-thread-pool-size=8
+spring.data.redis.lettuce.client.request-queue-size=1024
+
+spring.data.redis.username=default
+spring.data.redis.password=
+
+# Standalone mode
+spring.data.redis.host=localhost
+spring.data.redis.port=6379
+
+# Cluster mode: Cluster mode is prior than Standalone
+spring.data.redis.cluster.nodes=
diff --git a/collector/src/main/resources/profiles/release/redis.properties b/collector/src/main/resources/profiles/release/redis.properties
@@ -0,0 +1,13 @@
+spring.data.redis.lettuce.client.io-thread-pool-size=8
+spring.data.redis.lettuce.client.computation-thread-pool-size=8
+spring.data.redis.lettuce.client.request-queue-size=1024
+
+spring.data.redis.username=default
+spring.data.redis.password=
+
+# Standalone mode
+spring.data.redis.host=localhost
+spring.data.redis.port=6379
+
+# Cluster mode: Cluster mode is prior than Standalone
+spring.data.redis.cluster.nodes=
diff --git a/collector/src/main/resources/redis-root.properties b/collector/src/main/resources/redis-root.properties
@@ -0,0 +1 @@
+spring.data.redis.lettuce.client.name=collectorRedis
diff --git a/metric-module/collector-starter/pom.xml b/metric-module/collector-starter/pom.xml
@@ -34,6 +34,10 @@
             <groupId>com.navercorp.pinpoint</groupId>
             <artifactId>pinpoint-uristat-collector</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.navercorp.pinpoint</groupId>
+            <artifactId>pinpoint-realtime-collector</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/...in/java/com/navercorp/pinpoint/collector/starter/multi/application/BasicCollectorApp.java b/...in/java/com/navercorp/pinpoint/collector/starter/multi/application/BasicCollectorApp.java
@@ -3,6 +3,10 @@
 import com.navercorp.pinpoint.collector.PinpointCollectorModule;
 import org.springframework.boot.SpringBootConfiguration;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisRepositoriesAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration;
 import org.springframework.boot.autoconfigure.sql.init.SqlInitializationAutoConfiguration;
@@ -12,9 +16,13 @@
 @SpringBootConfiguration
 @EnableAutoConfiguration(exclude = {
         DataSourceAutoConfiguration.class,
-        DataSourceTransactionManagerAutoConfiguration.class,
         TransactionAutoConfiguration.class,
-        SqlInitializationAutoConfiguration.class
+        SqlInitializationAutoConfiguration.class,
+        SpringDataWebAutoConfiguration.class,
+        RedisAutoConfiguration.class,
+        RedisRepositoriesAutoConfiguration.class,
+        RedisReactiveAutoConfiguration.class,
+        DataSourceTransactionManagerAutoConfiguration.class
 })
 @Import({PinpointCollectorModule.class})
 public class BasicCollectorApp {

diff --git a/...ain/java/com/navercorp/pinpoint/collector/starter/multi/application/MultiApplication.java b/...ain/java/com/navercorp/pinpoint/collector/starter/multi/application/MultiApplication.java
@@ -10,11 +10,16 @@
 import com.navercorp.pinpoint.metric.collector.CollectorTypeParser;
 import com.navercorp.pinpoint.metric.collector.MetricCollectorApp;
 import com.navercorp.pinpoint.metric.collector.TypeSet;
+import com.navercorp.pinpoint.realtime.collector.RealtimeCollectorConfig;
 import com.navercorp.pinpoint.uristat.collector.UriStatCollectorConfig;
 import org.springframework.boot.Banner;
 import org.springframework.boot.SpringBootConfiguration;
 import org.springframework.boot.WebApplicationType;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisRepositoriesAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration;
 import org.springframework.boot.autoconfigure.transaction.TransactionAutoConfiguration;
@@ -26,7 +31,11 @@
 @EnableAutoConfiguration(exclude = {
         DataSourceAutoConfiguration.class,
         DataSourceTransactionManagerAutoConfiguration.class,
-        TransactionAutoConfiguration.class
+        TransactionAutoConfiguration.class,
+        SpringDataWebAutoConfiguration.class,
+        RedisAutoConfiguration.class,
+        RedisRepositoriesAutoConfiguration.class,
+        RedisReactiveAutoConfiguration.class
 })
 public class MultiApplication {
     private static final ServerBootLogger logger = ServerBootLogger.getLogger(MultiApplication.class);
@@ -49,7 +58,11 @@ public static void main(String[] args) {
 
         if (types.hasType(CollectorType.BASIC)) {
             logger.info(String.format("Start %s collector", CollectorType.BASIC));
-            SpringApplicationBuilder collectorAppBuilder = createAppBuilder(builder, 15400, BasicCollectorApp.class, UriStatCollectorConfig.class);
+            SpringApplicationBuilder collectorAppBuilder = createAppBuilder(builder, 15400,
+                    BasicCollectorApp.class,
+                    UriStatCollectorConfig.class,
+                    RealtimeCollectorConfig.class
+            );
             collectorAppBuilder.listeners(new AdditionalProfileListener("metric"));
             collectorAppBuilder.listeners(new AdditionalProfileListener("uri"));
             collectorAppBuilder.build().run(args);

diff --git a/metric-module/web-starter/pom.xml b/metric-module/web-starter/pom.xml
@@ -44,6 +44,10 @@
             <groupId>com.navercorp.pinpoint</groupId>
             <artifactId>pinpoint-uristat-web</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.navercorp.pinpoint</groupId>
+            <artifactId>pinpoint-realtime-web</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/...e/web-starter/src/main/java/com/navercorp/pinpoint/web/starter/multi/MetricAndWebApp.java b/...e/web-starter/src/main/java/com/navercorp/pinpoint/web/starter/multi/MetricAndWebApp.java
@@ -24,8 +24,13 @@
 import com.navercorp.pinpoint.web.PinpointWebModule;
 import com.navercorp.pinpoint.web.WebApp;
 import com.navercorp.pinpoint.web.WebStarter;
+import com.navercorp.pinpoint.web.realtime.RealtimeWebConfig;
 import org.springframework.boot.SpringBootConfiguration;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisRepositoriesAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
@@ -39,15 +44,26 @@
         DataSourceAutoConfiguration.class,
         DataSourceTransactionManagerAutoConfiguration.class,
         TransactionAutoConfiguration.class,
-        SecurityAutoConfiguration.class
+        SecurityAutoConfiguration.class,
+        SpringDataWebAutoConfiguration.class,
+        RedisAutoConfiguration.class,
+        RedisRepositoriesAutoConfiguration.class,
+        RedisReactiveAutoConfiguration.class
 })
 @Import({PinpointWebModule.class})
 public class MetricAndWebApp {
     private static final ServerBootLogger logger = ServerBootLogger.getLogger(WebApp.class);
 
     public static void main(String[] args) {
         try {
-            WebStarter starter = new WebStarter(MetricAndWebApp.class, PinpointBasicLoginConfig.class, AuthorizationConfig.class, MetricWebApp.class, UriStatWebConfig.class);
+            WebStarter starter = new WebStarter(
+                    MetricAndWebApp.class,
+                    PinpointBasicLoginConfig.class,
+                    AuthorizationConfig.class,
+                    MetricWebApp.class,
+                    UriStatWebConfig.class,
+                    RealtimeWebConfig.class
+            );
             starter.addProfiles("uri", "metric");
             starter.start(args);
         } catch (Exception exception) {

diff --git a/pom.xml b/pom.xml
@@ -121,6 +121,8 @@
         <module>pinot</module>
         <module>metric-module</module>
         <module>uristat</module>
+        <module>realtime</module>
+        <module>redis</module>
 
         <!-- <module>agent-testweb</module> -->
         <!-- <module>plugins-it</module> -->
@@ -333,6 +335,11 @@
                 <artifactId>pinpoint-rpc</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>com.navercorp.pinpoint</groupId>
+                <artifactId>pinpoint-redis</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>com.navercorp.pinpoint</groupId>
                 <artifactId>pinpoint-thrift</artifactId>
@@ -447,6 +454,21 @@
                 <artifactId>pinpoint-uristat-collector</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>com.navercorp.pinpoint</groupId>
+                <artifactId>pinpoint-realtime-web</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.navercorp.pinpoint</groupId>
+                <artifactId>pinpoint-realtime-collector</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.navercorp.pinpoint</groupId>
+                <artifactId>pinpoint-realtime-common</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>com.navercorp.pinpoint</groupId>
                 <artifactId>pinpoint-metric</artifactId>

diff --git a/realtime/pom.xml b/realtime/pom.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>pinpoint</artifactId>
+        <groupId>com.navercorp.pinpoint</groupId>
+        <version>2.6.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>pinpoint-realtime</artifactId>
+    <packaging>pom</packaging>
+
+    <modules>
+        <module>realtime-web</module>
+        <module>realtime-common</module>
+        <module>realtime-collector</module>
+    </modules>
+
+</project>
diff --git a/realtime/realtime-collector/pom.xml b/realtime/realtime-collector/pom.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>pinpoint-realtime</artifactId>
+        <groupId>com.navercorp.pinpoint</groupId>
+        <version>2.6.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>pinpoint-realtime-collector</artifactId>
+
+    <properties>
+        <jdk.version>11</jdk.version>
+        <jdk.home>${env.JAVA_11_HOME}</jdk.home>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.navercorp.pinpoint</groupId>
+            <artifactId>pinpoint-realtime-common</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.navercorp.pinpoint</groupId>
+            <artifactId>pinpoint-collector</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>
diff --git a/...ctor/src/main/java/com/navercorp/pinpoint/realtime/collector/RealtimeCollectorConfig.java b/...ctor/src/main/java/com/navercorp/pinpoint/realtime/collector/RealtimeCollectorConfig.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2023 NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.navercorp.pinpoint.realtime.collector;
+
+import com.navercorp.pinpoint.realtime.collector.activethread.count.CollectorActiveThreadCountConfig;
+import com.navercorp.pinpoint.realtime.collector.activethread.dump.CollectorActiveThreadDumpConfig;
+import com.navercorp.pinpoint.realtime.collector.echo.CollectorEchoConfig;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+
+/**
+ * @author youngjin.kim2
+ */
+@Configuration
+@ConditionalOnProperty(value = "pinpoint.collector.redis-realtime.enabled", havingValue = "true")
+@Import({
+        CollectorActiveThreadCountConfig.class,
+        CollectorActiveThreadDumpConfig.class,
+        CollectorEchoConfig.class
+})
+public class RealtimeCollectorConfig {
+}