Skip to content

Commit a3a709d

Browse files
merlimatmerlimat
committed
Merge remote-tracking branch 'apache/master' into metadata-session-reconnect
2 parents 3609b3d + bc44280 commit a3a709d

File tree

46 files changed

+2360
-1468
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+2360
-1468
lines changed

‎bin/pulsar-perf

+1-64
Original file line numberDiff line numberDiff line change
@@ -84,37 +84,6 @@ add_maven_deps_to_classpath() {
8484
fi
8585
PULSAR_CLASSPATH=${CLASSPATH}:`cat "${f}"`
8686
}
87-
pulsar_help() {
88-
cat <<EOF
89-
Usage: pulsar-perf <command>
90-
where command is one of:
91-
produce Run a producer
92-
consume Run a consumer
93-
transaction Run a transaction repeatedly
94-
read Run a topic reader
95-
96-
websocket-producer Run a websocket producer
97-
98-
managed-ledger Write directly on managed-ledgers
99-
monitor-brokers Continuously receive broker data and/or load reports
100-
simulation-client Run a simulation server acting as a Pulsar client
101-
simulation-controller Run a simulation controller to give commands to servers
102-
103-
gen-doc Generate documentation automatically.
104-
105-
help This help message
106-
107-
or command is the full name of a class with a defined main() method.
108-
109-
Environment variables:
110-
PULSAR_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF)
111-
PULSAR_CLIENT_CONF Configuration file for client (default: $DEFAULT_CLIENT_CONF)
112-
PULSAR_EXTRA_OPTS Extra options to be passed to the jvm
113-
PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath
114-
115-
These variable can also be set in conf/pulsar_env.sh
116-
EOF
117-
}
11887

11988
if [ -d "$PULSAR_HOME/lib" ]; then
12089
PULSAR_CLASSPATH="$PULSAR_CLASSPATH:$PULSAR_HOME/lib/*"
@@ -162,36 +131,4 @@ OPTS="$OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE"
162131
#Change to PULSAR_HOME to support relative paths
163132
cd "$PULSAR_HOME"
164133

165-
# if no args specified, show usage
166-
if [ $# = 0 ]; then
167-
pulsar_help;
168-
exit 1;
169-
fi
170-
171-
# get arguments
172-
COMMAND=$1
173-
shift
174-
175-
if [ "$COMMAND" == "produce" ]; then
176-
exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceProducer --conf-file $PULSAR_PERFTEST_CONF "$@"
177-
elif [ "$COMMAND" == "consume" ]; then
178-
exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceConsumer --conf-file $PULSAR_PERFTEST_CONF "$@"
179-
elif [ "$COMMAND" == "transaction" ]; then
180-
exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceTransaction --conf-file $PULSAR_PERFTEST_CONF "$@"
181-
elif [ "$COMMAND" == "read" ]; then
182-
exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceReader --conf-file $PULSAR_PERFTEST_CONF "$@"
183-
elif [ "$COMMAND" == "monitor-brokers" ]; then
184-
exec $JAVA $OPTS org.apache.pulsar.testclient.BrokerMonitor "$@"
185-
elif [ "$COMMAND" == "simulation-client" ]; then
186-
exec $JAVA $OPTS org.apache.pulsar.testclient.LoadSimulationClient "$@"
187-
elif [ "$COMMAND" == "simulation-controller" ]; then
188-
exec $JAVA $OPTS org.apache.pulsar.testclient.LoadSimulationController "$@"
189-
elif [ "$COMMAND" == "websocket-producer" ]; then
190-
exec $JAVA $OPTS org.apache.pulsar.proxy.socket.client.PerformanceClient "$@"
191-
elif [ "$COMMAND" == "managed-ledger" ]; then
192-
exec $JAVA $OPTS org.apache.pulsar.testclient.ManagedLedgerWriter "$@"
193-
elif [ "$COMMAND" == "gen-doc" ]; then
194-
exec $JAVA $OPTS org.apache.pulsar.testclient.CmdGenerateDocumentation "$@"
195-
else
196-
pulsar_help;
197-
fi
134+
exec $JAVA $OPTS org.apache.pulsar.testclient.PulsarPerfTestTool $PULSAR_PERFTEST_CONF "$@"

‎bin/pulsar-perf.cmd

+1-83
Original file line numberDiff line numberDiff line change
@@ -72,67 +72,7 @@ set "OPTS=%OPTS% -Dpulsar.log.level=%PULSAR_LOG_LEVEL%"
7272
set "OPTS=%OPTS% -Dpulsar.log.root.level=%PULSAR_LOG_ROOT_LEVEL%"
7373
set "OPTS=%OPTS% -Dpulsar.log.immediateFlush=%PULSAR_LOG_IMMEDIATE_FLUSH%"
7474

75-
set "COMMAND=%1"
76-
77-
for /f "tokens=1,* delims= " %%a in ("%*") do set "_args=%%b"
78-
79-
if "%COMMAND%" == "produce" (
80-
call :execCmdWithConfigFile org.apache.pulsar.testclient.PerformanceProducer
81-
exit /B %ERROR_CODE%
82-
)
83-
if "%COMMAND%" == "consume" (
84-
call :execCmdWithConfigFile org.apache.pulsar.testclient.PerformanceConsumer
85-
exit /B %ERROR_CODE%
86-
)
87-
if "%COMMAND%" == "transaction" (
88-
call :execCmdWithConfigFile org.apache.pulsar.testclient.PerformanceTransaction
89-
exit /B %ERROR_CODE%
90-
)
91-
if "%COMMAND%" == "read" (
92-
call :execCmdWithConfigFile org.apache.pulsar.testclient.PerformanceReader
93-
exit /B %ERROR_CODE%
94-
)
95-
if "%COMMAND%" == "monitor-brokers" (
96-
call :execCmd org.apache.pulsar.testclient.BrokerMonitor
97-
exit /B %ERROR_CODE%
98-
)
99-
if "%COMMAND%" == "simulation-client" (
100-
call :execCmd org.apache.pulsar.testclient.LoadSimulationClient
101-
exit /B %ERROR_CODE%
102-
)
103-
if "%COMMAND%" == "simulation-controller" (
104-
call :execCmd org.apache.pulsar.testclient.LoadSimulationController
105-
exit /B %ERROR_CODE%
106-
)
107-
if "%COMMAND%" == "websocket-producer" (
108-
call :execCmd org.apache.pulsar.proxy.socket.client.PerformanceClient
109-
exit /B %ERROR_CODE%
110-
)
111-
if "%COMMAND%" == "managed-ledger" (
112-
call :execCmd org.apache.pulsar.testclient.ManagedLedgerWriter
113-
exit /B %ERROR_CODE%
114-
)
115-
if "%COMMAND%" == "gen-doc" (
116-
call :execCmd org.apache.pulsar.testclient.CmdGenerateDocumentation
117-
exit /B %ERROR_CODE%
118-
)
119-
120-
call :usage
121-
exit /B %ERROR_CODE%
122-
123-
:execCmdWithConfigFile
124-
"%JAVACMD%" %OPTS% %1 --conf-file "%PULSAR_PERFTEST_CONF%" %_args%
125-
if ERRORLEVEL 1 (
126-
call :error
127-
)
128-
goto :eof
129-
130-
:execCmd
131-
"%JAVACMD%" %OPTS% %1 %_args%
132-
if ERRORLEVEL 1 (
133-
call :error
134-
)
135-
goto :eof
75+
"%JAVACMD%" %OPTS% org.apache.pulsar.testclient.PulsarPerfTestTool "%PULSAR_PERFTEST_CONF%" %*
13676

13777

13878

@@ -142,25 +82,3 @@ goto :eof
14282

14383

14484

145-
146-
:usage
147-
echo Usage: pulsar-perf COMMAND
148-
echo where command is one of:
149-
echo produce Run a producer
150-
echo consume Run a consumer
151-
echo transaction Run a transaction repeatedly
152-
echo read Run a topic reader
153-
echo websocket-producer Run a websocket producer
154-
echo managed-ledger Write directly on managed-ledgers
155-
echo monitor-brokers Continuously receive broker data and/or load reports
156-
echo simulation-client Run a simulation server acting as a Pulsar client
157-
echo simulation-controller Run a simulation controller to give commands to servers
158-
echo gen-doc Generate documentation automatically.
159-
echo help This help message
160-
echo or command is the full name of a class with a defined main() method.
161-
echo Environment variables:
162-
echo PULSAR_LOG_CONF Log4j configuration file (default %PULSAR_HOME%\logs)
163-
echo PULSAR_CLIENT_CONF Configuration file for client (default: %PULSAR_HOME%\conf\client.conf)
164-
echo PULSAR_EXTRA_OPTS Extra options to be passed to the jvm
165-
echo PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath
166-
goto error

‎managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java

+10
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,11 @@ public interface ManagedLedgerMXBean {
8585
*/
8686
long getAddEntrySucceed();
8787

88+
/**
89+
* @return the total number of addEntry requests that succeeded
90+
*/
91+
long getAddEntrySucceedTotal();
92+
8893
/**
8994
* @return the number of addEntry requests that failed
9095
*/
@@ -100,6 +105,11 @@ public interface ManagedLedgerMXBean {
100105
*/
101106
long getReadEntriesSucceeded();
102107

108+
/**
109+
* @return the total number of readEntries requests that succeeded
110+
*/
111+
long getReadEntriesSucceededTotal();
112+
103113
/**
104114
* @return the number of readEntries requests that failed
105115
*/

‎managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java

+10
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,11 @@ public long getAddEntrySucceed() {
230230
return addEntryOps.getCount();
231231
}
232232

233+
@Override
234+
public long getAddEntrySucceedTotal() {
235+
return addEntryOps.getTotalCount();
236+
}
237+
233238
@Override
234239
public long getAddEntryErrors() {
235240
return addEntryOpsFailed.getCount();
@@ -240,6 +245,11 @@ public long getReadEntriesSucceeded() {
240245
return readEntriesOps.getCount();
241246
}
242247

248+
@Override
249+
public long getReadEntriesSucceededTotal() {
250+
return readEntriesOps.getTotalCount();
251+
}
252+
243253
@Override
244254
public long getReadEntriesErrors() {
245255
return readEntriesOpsFailed.getCount();

‎managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanTest.java

+8
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,12 @@ public void simple() throws Exception {
7777
assertEquals(mbean.getAddEntryWithReplicasBytesRate(), 0.0);
7878
assertEquals(mbean.getAddEntryMessagesRate(), 0.0);
7979
assertEquals(mbean.getAddEntrySucceed(), 0);
80+
assertEquals(mbean.getAddEntrySucceedTotal(), 0);
8081
assertEquals(mbean.getAddEntryErrors(), 0);
8182
assertEquals(mbean.getReadEntriesBytesRate(), 0.0);
8283
assertEquals(mbean.getReadEntriesRate(), 0.0);
8384
assertEquals(mbean.getReadEntriesSucceeded(), 0);
85+
assertEquals(mbean.getReadEntriesSucceededTotal(), 0);
8486
assertEquals(mbean.getReadEntriesErrors(), 0);
8587
assertEquals(mbean.getMarkDeleteRate(), 0.0);
8688

@@ -105,10 +107,12 @@ public void simple() throws Exception {
105107
assertEquals(mbean.getAddEntryWithReplicasBytesRate(), 1600.0);
106108
assertEquals(mbean.getAddEntryMessagesRate(), 2.0);
107109
assertEquals(mbean.getAddEntrySucceed(), 2);
110+
assertEquals(mbean.getAddEntrySucceedTotal(), 2);
108111
assertEquals(mbean.getAddEntryErrors(), 0);
109112
assertEquals(mbean.getReadEntriesBytesRate(), 0.0);
110113
assertEquals(mbean.getReadEntriesRate(), 0.0);
111114
assertEquals(mbean.getReadEntriesSucceeded(), 0);
115+
assertEquals(mbean.getReadEntriesSucceededTotal(), 0);
112116
assertEquals(mbean.getReadEntriesErrors(), 0);
113117
assertTrue(mbean.getMarkDeleteRate() > 0.0);
114118

@@ -134,10 +138,14 @@ public void simple() throws Exception {
134138
assertEquals(mbean.getReadEntriesBytesRate(), 600.0);
135139
assertEquals(mbean.getReadEntriesRate(), 1.0);
136140
assertEquals(mbean.getReadEntriesSucceeded(), 1);
141+
assertEquals(mbean.getReadEntriesSucceededTotal(), 1);
137142
assertEquals(mbean.getReadEntriesErrors(), 0);
138143
assertEquals(mbean.getNumberOfMessagesInBacklog(), 1);
139144
assertEquals(mbean.getMarkDeleteRate(), 0.0);
140145

146+
assertEquals(mbean.getAddEntrySucceed(), 0);
147+
assertEquals(mbean.getAddEntrySucceedTotal(), 2);
148+
141149
factory.shutdown();
142150
}
143151

‎pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ flexible messaging model and an intuitive client API.</description>
182182
<aerospike-client.version>4.5.0</aerospike-client.version>
183183
<kafka-client.version>3.4.0</kafka-client.version>
184184
<rabbitmq-client.version>5.18.0</rabbitmq-client.version>
185-
<aws-sdk.version>1.12.262</aws-sdk.version>
185+
<aws-sdk.version>1.12.638</aws-sdk.version>
186186
<avro.version>1.11.3</avro.version>
187187
<joda.version>2.10.10</joda.version>
188188
<jclouds.version>2.6.0</jclouds.version>

‎pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java

+8
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@
109109
import org.apache.pulsar.broker.service.schema.SchemaRegistryService;
110110
import org.apache.pulsar.broker.service.schema.SchemaStorageFactory;
111111
import org.apache.pulsar.broker.stats.MetricsGenerator;
112+
import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats;
112113
import org.apache.pulsar.broker.stats.PulsarBrokerOpenTelemetry;
113114
import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsServlet;
114115
import org.apache.pulsar.broker.stats.prometheus.PrometheusRawMetricsProvider;
@@ -252,6 +253,7 @@ public class PulsarService implements AutoCloseable, ShutdownService {
252253

253254
private MetricsGenerator metricsGenerator;
254255
private final PulsarBrokerOpenTelemetry openTelemetry;
256+
private OpenTelemetryTopicStats openTelemetryTopicStats;
255257

256258
private TransactionMetadataStoreService transactionMetadataStoreService;
257259
private TransactionBufferProvider transactionBufferProvider;
@@ -631,6 +633,10 @@ public CompletableFuture<Void> closeAsync() {
631633
brokerClientSharedTimer.stop();
632634
monotonicSnapshotClock.close();
633635

636+
if (openTelemetryTopicStats != null) {
637+
openTelemetryTopicStats.close();
638+
}
639+
634640
asyncCloseFutures.add(EventLoopUtil.shutdownGracefully(ioEventLoopGroup));
635641

636642

@@ -771,6 +777,8 @@ public void start() throws PulsarServerException {
771777
config.getDefaultRetentionTimeInMinutes() * 60));
772778
}
773779

780+
openTelemetryTopicStats = new OpenTelemetryTopicStats(this);
781+
774782
localMetadataSynchronizer = StringUtils.isNotBlank(config.getMetadataSyncEventTopic())
775783
? new PulsarMetadataEventSynchronizer(this, config.getMetadataSyncEventTopic())
776784
: null;

‎pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java

+8
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,9 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener<TopicP
133133
private static final AtomicLongFieldUpdater<AbstractTopic> RATE_LIMITED_UPDATER =
134134
AtomicLongFieldUpdater.newUpdater(AbstractTopic.class, "publishRateLimitedTimes");
135135
protected volatile long publishRateLimitedTimes = 0L;
136+
private static final AtomicLongFieldUpdater<AbstractTopic> TOTAL_RATE_LIMITED_UPDATER =
137+
AtomicLongFieldUpdater.newUpdater(AbstractTopic.class, "totalPublishRateLimitedCounter");
138+
protected volatile long totalPublishRateLimitedCounter = 0L;
136139

137140
private static final AtomicIntegerFieldUpdater<AbstractTopic> USER_CREATED_PRODUCER_COUNTER_UPDATER =
138141
AtomicIntegerFieldUpdater.newUpdater(AbstractTopic.class, "userCreatedProducerCount");
@@ -897,6 +900,7 @@ public void recordAddLatency(long latency, TimeUnit unit) {
897900

898901
@Override
899902
public long increasePublishLimitedTimes() {
903+
TOTAL_RATE_LIMITED_UPDATER.incrementAndGet(this);
900904
return RATE_LIMITED_UPDATER.incrementAndGet(this);
901905
}
902906

@@ -1185,6 +1189,10 @@ public long getBytesOutCounter() {
11851189
+ sumSubscriptions(AbstractSubscription::getBytesOutCounter);
11861190
}
11871191

1192+
public long getTotalPublishRateLimitCounter() {
1193+
return TOTAL_RATE_LIMITED_UPDATER.get(this);
1194+
}
1195+
11881196
private long sumSubscriptions(ToLongFunction<AbstractSubscription> toCounter) {
11891197
return getSubscriptions().values().stream()
11901198
.map(AbstractSubscription.class::cast)

0 commit comments

Comments
 (0)