Skip to content

Commit 3169386

Browse files
author
youngjin.kim2
committed
[pinpoint-apm#9666] Realtime activeThreadCount with Redis
1 parent f52be14 commit 3169386

File tree

54 files changed

+2880
-40
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+2880
-40
lines changed

‎collector/src/main/java/com/navercorp/pinpoint/collector/cluster/route/AbstractRouteHandler.java

+5-3
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,8 @@
2222
import com.navercorp.pinpoint.thrift.dto.command.TCommandTransfer;
2323
import com.navercorp.pinpoint.thrift.dto.command.TCommandTransferResponse;
2424
import com.navercorp.pinpoint.thrift.dto.command.TRouteResult;
25-
26-
import org.apache.logging.log4j.Logger;
2725
import org.apache.logging.log4j.LogManager;
26+
import org.apache.logging.log4j.Logger;
2827

2928
import java.util.ArrayList;
3029
import java.util.List;
@@ -47,7 +46,10 @@ protected ClusterPoint<?> findClusterPoint(TCommandTransfer deliveryCommand) {
4746
String agentId = deliveryCommand.getAgentId();
4847
long startTimeStamp = deliveryCommand.getStartTime();
4948
final ClusterKey sourceKey = new ClusterKey(applicationName, agentId, startTimeStamp);
49+
return findClusterPoint(sourceKey);
50+
}
5051

52+
public ClusterPoint<?> findClusterPoint(ClusterKey sourceKey) {
5153
List<ClusterPoint<?>> result = new ArrayList<>();
5254

5355
for (ClusterPoint<?> targetClusterPoint : targetClusterPointLocator.getClusterPointList()) {
@@ -62,7 +64,7 @@ protected ClusterPoint<?> findClusterPoint(TCommandTransfer deliveryCommand) {
6264
}
6365

6466
if (result.size() > 1) {
65-
logger.warn("Ambiguous ClusterPoint {}, {}, {} (Valid Agent list={}).", applicationName, agentId, startTimeStamp, result);
67+
logger.warn("Ambiguous ClusterPoint {} (Valid Agent list={}).", sourceKey, result);
6668
return null;
6769
}
6870

‎metric-module/collector-starter/pom.xml

+4
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@
3434
<groupId>com.navercorp.pinpoint</groupId>
3535
<artifactId>pinpoint-uristat-collector</artifactId>
3636
</dependency>
37+
<dependency>
38+
<groupId>com.navercorp.pinpoint</groupId>
39+
<artifactId>pinpoint-realtime-collector</artifactId>
40+
</dependency>
3741
</dependencies>
3842

3943
<build>

‎metric-module/collector-starter/src/main/java/com/navercorp/pinpoint/collector/starter/multi/application/BasicCollectorApp.java

+13-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@
44
import com.navercorp.pinpoint.collector.config.FlinkContextConfiguration;
55
import org.springframework.boot.SpringBootConfiguration;
66
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
7+
import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
8+
import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
9+
import org.springframework.boot.autoconfigure.data.redis.RedisRepositoriesAutoConfiguration;
10+
import org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration;
711
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
812
import org.springframework.boot.autoconfigure.sql.init.SqlInitializationAutoConfiguration;
913
import org.springframework.boot.autoconfigure.transaction.TransactionAutoConfiguration;
@@ -12,7 +16,15 @@
1216
import org.springframework.context.annotation.ImportResource;
1317

1418
@SpringBootConfiguration
15-
@EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class, TransactionAutoConfiguration.class, SqlInitializationAutoConfiguration.class})
19+
@EnableAutoConfiguration(exclude = {
20+
DataSourceAutoConfiguration.class,
21+
TransactionAutoConfiguration.class,
22+
SqlInitializationAutoConfiguration.class,
23+
SpringDataWebAutoConfiguration.class,
24+
RedisAutoConfiguration.class,
25+
RedisRepositoriesAutoConfiguration.class,
26+
RedisReactiveAutoConfiguration.class
27+
})
1628
@ImportResource({"classpath:applicationContext-collector.xml", "classpath:servlet-context-collector.xml"})
1729
@Import({CollectorAppPropertySources.class, FlinkContextConfiguration.class})
1830
@ComponentScan({})

‎metric-module/collector-starter/src/main/java/com/navercorp/pinpoint/collector/starter/multi/application/MultiApplication.java

+18-2
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,31 @@
1010
import com.navercorp.pinpoint.metric.collector.CollectorTypeParser;
1111
import com.navercorp.pinpoint.metric.collector.MetricCollectorApp;
1212
import com.navercorp.pinpoint.metric.collector.TypeSet;
13+
import com.navercorp.pinpoint.realtime.collector.atc.config.RealtimeCollectorConfig;
1314
import com.navercorp.pinpoint.uristat.collector.UriStatCollectorConfig;
1415
import org.springframework.boot.Banner;
1516
import org.springframework.boot.SpringBootConfiguration;
1617
import org.springframework.boot.WebApplicationType;
1718
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
19+
import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
20+
import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
21+
import org.springframework.boot.autoconfigure.data.redis.RedisRepositoriesAutoConfiguration;
22+
import org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration;
1823
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
1924
import org.springframework.boot.autoconfigure.transaction.TransactionAutoConfiguration;
2025
import org.springframework.boot.builder.SpringApplicationBuilder;
2126

2227
import java.util.Arrays;
2328

2429
@SpringBootConfiguration
25-
@EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class, TransactionAutoConfiguration.class})
30+
@EnableAutoConfiguration(exclude = {
31+
DataSourceAutoConfiguration.class,
32+
TransactionAutoConfiguration.class,
33+
SpringDataWebAutoConfiguration.class,
34+
RedisAutoConfiguration.class,
35+
RedisRepositoriesAutoConfiguration.class,
36+
RedisReactiveAutoConfiguration.class
37+
})
2638
public class MultiApplication {
2739
private static final ServerBootLogger logger = ServerBootLogger.getLogger(MultiApplication.class);
2840

@@ -44,7 +56,11 @@ public static void main(String[] args) {
4456

4557
if (types.hasType(CollectorType.BASIC)) {
4658
logger.info(String.format("Start %s collector", CollectorType.BASIC));
47-
SpringApplicationBuilder collectorAppBuilder = createAppBuilder(builder, 15400, BasicCollectorApp.class, UriStatCollectorConfig.class);
59+
SpringApplicationBuilder collectorAppBuilder = createAppBuilder(builder, 15400,
60+
BasicCollectorApp.class,
61+
UriStatCollectorConfig.class,
62+
RealtimeCollectorConfig.class
63+
);
4864
collectorAppBuilder.build().run(args);
4965
}
5066

‎metric-module/web-starter/pom.xml

+4
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@
4444
<groupId>com.navercorp.pinpoint</groupId>
4545
<artifactId>pinpoint-uristat-web</artifactId>
4646
</dependency>
47+
<dependency>
48+
<groupId>com.navercorp.pinpoint</groupId>
49+
<artifactId>pinpoint-realtime-web</artifactId>
50+
</dependency>
4751
</dependencies>
4852

4953
<build>

‎metric-module/web-starter/src/main/java/com/navercorp/pinpoint/web/starter/multi/MetricAndWebApp.java

+23-4
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,22 @@
1818

1919
import com.navercorp.pinpoint.common.server.util.ServerBootLogger;
2020
import com.navercorp.pinpoint.metric.web.MetricWebApp;
21+
import com.navercorp.pinpoint.realtime.web.atc.config.RealtimeWebConfig;
2122
import com.navercorp.pinpoint.uristat.web.UriStatWebConfig;
23+
import com.navercorp.pinpoint.web.AuthorizationConfig;
2224
import com.navercorp.pinpoint.web.PinpointBasicLoginConfig;
2325
import com.navercorp.pinpoint.web.WebApp;
2426
import com.navercorp.pinpoint.web.WebAppPropertySources;
2527
import com.navercorp.pinpoint.web.WebMvcConfig;
2628
import com.navercorp.pinpoint.web.WebServerConfig;
2729
import com.navercorp.pinpoint.web.WebStarter;
28-
import com.navercorp.pinpoint.web.AuthorizationConfig;
2930
import com.navercorp.pinpoint.web.cache.CacheConfiguration;
3031
import org.springframework.boot.SpringBootConfiguration;
3132
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
33+
import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
34+
import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
35+
import org.springframework.boot.autoconfigure.data.redis.RedisRepositoriesAutoConfiguration;
36+
import org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration;
3237
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
3338
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
3439
import org.springframework.boot.autoconfigure.transaction.TransactionAutoConfiguration;
@@ -38,16 +43,30 @@
3843
* @author minwoo.jung
3944
*/
4045
@SpringBootConfiguration
41-
@EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class, TransactionAutoConfiguration.class,
42-
SecurityAutoConfiguration.class})
46+
@EnableAutoConfiguration(exclude = {
47+
DataSourceAutoConfiguration.class,
48+
TransactionAutoConfiguration.class,
49+
SecurityAutoConfiguration.class,
50+
SpringDataWebAutoConfiguration.class,
51+
RedisAutoConfiguration.class,
52+
RedisRepositoriesAutoConfiguration.class,
53+
RedisReactiveAutoConfiguration.class
54+
})
4355
@ImportResource({"classpath:applicationContext-web.xml", "classpath:servlet-context-web.xml"})
4456
@Import({WebAppPropertySources.class, WebServerConfig.class, WebMvcConfig.class, CacheConfiguration.class})
4557
public class MetricAndWebApp {
4658
private static final ServerBootLogger logger = ServerBootLogger.getLogger(WebApp.class);
4759

4860
public static void main(String[] args) {
4961
try {
50-
WebStarter starter = new WebStarter(MetricAndWebApp.class, PinpointBasicLoginConfig.class, AuthorizationConfig.class, MetricWebApp.class, UriStatWebConfig.class);
62+
WebStarter starter = new WebStarter(
63+
MetricAndWebApp.class,
64+
PinpointBasicLoginConfig.class,
65+
AuthorizationConfig.class,
66+
MetricWebApp.class,
67+
UriStatWebConfig.class,
68+
RealtimeWebConfig.class
69+
);
5170
starter.start(args);
5271
} catch (Exception exception) {
5372
logger.error("[WebApp] could not launch app.", exception);

‎pom.xml

+16
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@
119119
<module>hbase2-module</module>
120120
<module>metric-module</module>
121121
<module>uristat</module>
122+
<module>realtime</module>
122123

123124
<!-- <module>agent-testweb</module> -->
124125
<!-- <module>plugins-it</module> -->
@@ -439,6 +440,21 @@
439440
<artifactId>pinpoint-uristat-collector</artifactId>
440441
<version>${project.version}</version>
441442
</dependency>
443+
<dependency>
444+
<groupId>com.navercorp.pinpoint</groupId>
445+
<artifactId>pinpoint-realtime-web</artifactId>
446+
<version>${project.version}</version>
447+
</dependency>
448+
<dependency>
449+
<groupId>com.navercorp.pinpoint</groupId>
450+
<artifactId>pinpoint-realtime-collector</artifactId>
451+
<version>${project.version}</version>
452+
</dependency>
453+
<dependency>
454+
<groupId>com.navercorp.pinpoint</groupId>
455+
<artifactId>pinpoint-realtime-common</artifactId>
456+
<version>${project.version}</version>
457+
</dependency>
442458
<dependency>
443459
<groupId>com.navercorp.pinpoint</groupId>
444460
<artifactId>pinpoint-metric</artifactId>

‎realtime/pom.xml

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>pinpoint</artifactId>
7+
<groupId>com.navercorp.pinpoint</groupId>
8+
<version>2.5.1-SNAPSHOT</version>
9+
</parent>
10+
<modelVersion>4.0.0</modelVersion>
11+
12+
<artifactId>pinpoint-realtime</artifactId>
13+
<packaging>pom</packaging>
14+
15+
<modules>
16+
<module>realtime-web</module>
17+
<module>realtime-common</module>
18+
<module>realtime-collector</module>
19+
</modules>
20+
21+
</project>

‎realtime/realtime-collector/pom.xml

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>pinpoint-realtime</artifactId>
7+
<groupId>com.navercorp.pinpoint</groupId>
8+
<version>2.5.1-SNAPSHOT</version>
9+
</parent>
10+
<modelVersion>4.0.0</modelVersion>
11+
12+
<artifactId>pinpoint-realtime-collector</artifactId>
13+
14+
<properties>
15+
<jdk.version>11</jdk.version>
16+
<jdk.home>${env.JAVA_11_HOME}</jdk.home>
17+
</properties>
18+
19+
<dependencies>
20+
<dependency>
21+
<groupId>com.navercorp.pinpoint</groupId>
22+
<artifactId>pinpoint-realtime-common</artifactId>
23+
</dependency>
24+
<dependency>
25+
<groupId>com.navercorp.pinpoint</groupId>
26+
<artifactId>pinpoint-collector</artifactId>
27+
</dependency>
28+
</dependencies>
29+
30+
</project>

‎realtime/realtime-collector/src/main/java/com/navercorp/pinpoint/realtime/collector/atc/config/RealtimeCollectorConfig.java

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*
2+
* Copyright 2023 NAVER Corp.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.navercorp.pinpoint.realtime.collector.atc.config;
17+
18+
import com.navercorp.pinpoint.collector.cluster.route.StreamRouteHandler;
19+
import com.navercorp.pinpoint.realtime.ActiveThreadCountSupplyTopic;
20+
import com.navercorp.pinpoint.realtime.collector.atc.redis.ActiveThreadCountDemandListener;
21+
import com.navercorp.pinpoint.realtime.collector.atc.service.ATCRequestService;
22+
import com.navercorp.pinpoint.realtime.collector.atc.service.netty.ATCPacketDeserializer;
23+
import com.navercorp.pinpoint.realtime.collector.atc.service.netty.NettyATCRequestService;
24+
import com.navercorp.pinpoint.realtime.collector.atc.service.redis.RedisATCPublisher;
25+
import com.navercorp.pinpoint.thrift.io.DeserializerFactory;
26+
import com.navercorp.pinpoint.thrift.io.HeaderTBaseDeserializer;
27+
import com.navercorp.pinpoint.thrift.io.HeaderTBaseSerializer;
28+
import com.navercorp.pinpoint.thrift.io.SerializerFactory;
29+
import org.springframework.beans.factory.annotation.Qualifier;
30+
import org.springframework.beans.factory.annotation.Value;
31+
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
32+
import org.springframework.context.annotation.Bean;
33+
import org.springframework.context.annotation.Configuration;
34+
import org.springframework.context.annotation.Import;
35+
import org.springframework.data.redis.connection.RedisConnectionFactory;
36+
import org.springframework.data.redis.listener.RedisMessageListenerContainer;
37+
38+
/**
39+
* @author youngjin.kim2
40+
*/
41+
@Configuration
42+
@Import(RedisCollectorConfig.class)
43+
public class RealtimeCollectorConfig {
44+
45+
@Value("${pinpoint.collector.realtime.atc.demand.maxAgeMs:12500}")
46+
long demandMaxAgeMs;
47+
48+
@Value("${pinpoint.collector.realtime.atc.throttle.termMillis:100}")
49+
long throttleTermMillis;
50+
51+
@Bean
52+
@ConditionalOnBean(RedisConnectionFactory.class)
53+
ActiveThreadCountSupplyTopic activeThreadCountSupplyTopic(RedisConnectionFactory redisConnectionFactory) {
54+
return new ActiveThreadCountSupplyTopic(redisConnectionFactory);
55+
}
56+
57+
@Bean
58+
@ConditionalOnBean(ActiveThreadCountSupplyTopic.class)
59+
RedisATCPublisher redisActiveThreadCountPublisher(ActiveThreadCountSupplyTopic topic) {
60+
return new RedisATCPublisher(topic, throttleTermMillis);
61+
}
62+
63+
@Bean
64+
@ConditionalOnBean(name = "commandHeaderTBaseDeserializerFactory")
65+
ATCPacketDeserializer atcPacketDeserializer(
66+
@Qualifier("commandHeaderTBaseDeserializerFactory")
67+
DeserializerFactory<HeaderTBaseDeserializer> deserializerFactory
68+
) {
69+
return new ATCPacketDeserializer(deserializerFactory);
70+
}
71+
72+
@Bean
73+
@ConditionalOnBean(value = { StreamRouteHandler.class }, name = "commandHeaderTBaseSerializerFactory")
74+
ATCRequestService atcService(
75+
StreamRouteHandler streamRouteHandler,
76+
@Qualifier("commandHeaderTBaseSerializerFactory")
77+
SerializerFactory<HeaderTBaseSerializer> serializerFactory,
78+
ATCPacketDeserializer deserializer
79+
) {
80+
return new NettyATCRequestService(streamRouteHandler, serializerFactory, deserializer);
81+
}
82+
83+
@Bean
84+
@ConditionalOnBean({ RedisMessageListenerContainer.class })
85+
ActiveThreadCountDemandListener activeThreadCountDemandListener(
86+
RedisMessageListenerContainer container,
87+
ATCRequestService atcRequestService,
88+
RedisATCPublisher publisher
89+
) {
90+
return new ActiveThreadCountDemandListener(container, atcRequestService, publisher, demandMaxAgeMs);
91+
}
92+
93+
}

0 commit comments

Comments
 (0)