From 0d6fcce274221f43d70ae247c672b6ac8c0ce9ef Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Wed, 17 Jan 2024 10:41:56 -0800 Subject: [PATCH 01/28] heartbeat metrics --- .../internals/HeartbeatRequestManager.java | 35 +++++++++++++++++-- .../consumer/internals/RequestManagers.java | 3 +- .../metrics/AbstractConsumerMetrics.java | 14 ++++++++ .../internals/metrics/HeartbeatMetrics.java | 31 ++++++++++++++++ .../internals/AsyncKafkaConsumerTest.java | 19 ++++++++++ .../internals/ConsumerTestBuilder.java | 4 ++- .../HeartbeatRequestManagerTest.java | 29 ++++++++------- 7 files changed, 118 insertions(+), 17 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 11846eedeafdc..04b342fa9a0a5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -23,10 +23,13 @@ import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.GroupMetadataUpdateEvent; +import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetrics; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.GroupAuthorizationException; import org.apache.kafka.common.errors.RetriableException; import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; @@ -41,6 +44,7 @@ import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -78,6 +82,11 @@ public class HeartbeatRequestManager implements RequestManager { */ private final int maxPollIntervalMs; + /** + * The last time in Ms the heartbeat was sent to the coordinator + */ + private long lastHeartbeatSend = 0L; + /** * CoordinatorRequestManager manages the connection to the group coordinator */ @@ -108,6 +117,7 @@ public class HeartbeatRequestManager implements RequestManager { * sending heartbeat until the next poll. */ private final Timer pollTimer; + private final HeartbeatMetrics heartbeatMetrics; private GroupMetadataUpdateEvent previousGroupMetadataUpdateEvent = null; @@ -118,7 +128,8 @@ public HeartbeatRequestManager( final CoordinatorRequestManager coordinatorRequestManager, final SubscriptionState subscriptions, final MembershipManager membershipManager, - final BackgroundEventHandler backgroundEventHandler) { + final BackgroundEventHandler backgroundEventHandler, + final Metrics metrics) { this.coordinatorRequestManager = coordinatorRequestManager; this.logger = logContext.logger(getClass()); this.membershipManager = membershipManager; @@ -130,6 +141,8 @@ public HeartbeatRequestManager( this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, retryBackoffMaxMs, maxPollIntervalMs); this.pollTimer = time.timer(maxPollIntervalMs); + this.heartbeatMetrics = new HeartbeatMetrics(metrics); + registerLastSentHeartbeatMetric(metrics); } // Visible for testing @@ -141,7 +154,8 @@ public HeartbeatRequestManager( final MembershipManager membershipManager, final HeartbeatState heartbeatState, final HeartbeatRequestState heartbeatRequestState, - final BackgroundEventHandler backgroundEventHandler) { + final BackgroundEventHandler backgroundEventHandler, + final Metrics metrics) { this.logger = logContext.logger(this.getClass()); this.maxPollIntervalMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); this.coordinatorRequestManager = coordinatorRequestManager; @@ -150,6 +164,8 @@ public HeartbeatRequestManager( this.membershipManager = membershipManager; this.backgroundEventHandler = backgroundEventHandler; this.pollTimer = timer; + this.heartbeatMetrics = new HeartbeatMetrics(metrics); + registerLastSentHeartbeatMetric(metrics); } /** @@ -245,6 +261,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final long curr NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(ignoreResponse); heartbeatRequestState.onSendAttempt(currentTimeMs); membershipManager.onHeartbeatRequestSent(); + lastHeartbeatSend = currentTimeMs; return request; } @@ -415,6 +432,20 @@ private void handleFatalFailure(Throwable error) { membershipManager.transitionToFatal(); } + private void registerLastSentHeartbeatMetric(Metrics metrics) { + Measurable lastHeartbeat = (config, now) -> { + if (lastHeartbeatSend == 0L) + // if no heartbeat is ever triggered, just return -1. + return -1d; + else + return TimeUnit.SECONDS.convert(now - lastHeartbeatSend, TimeUnit.MILLISECONDS); + }; + heartbeatMetrics.registerMeasurable( + "heartbeat-time-max", + "The max time taken to receive a response to a heartbeat request", + lastHeartbeat); + } + /** * Represents the state of a heartbeat request, including logic for timing, retries, and exponential backoff. The * object extends {@link RequestState} to enable exponential backoff and duplicated request handling. The two fields diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index 6dd9e01216343..7b3e75be0182c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -190,7 +190,8 @@ protected RequestManagers create() { coordinator, subscriptions, membershipManager, - backgroundEventHandler); + backgroundEventHandler, + metrics); } return new RequestManagers( diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java index 2273bfd494a3b..c9cd16b26f8cd 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java @@ -16,6 +16,10 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.stats.Meter; +import org.apache.kafka.common.metrics.stats.WindowedCount; + import java.util.Optional; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; @@ -26,4 +30,14 @@ public abstract class AbstractConsumerMetrics { public AbstractConsumerMetrics(Optional grpMetricsPrefix) { grpMetricsPrefix.ifPresent(s -> this.groupMetricsName = s + COORDINATOR_METRICS_SUFFIX); } + + public AbstractConsumerMetrics() {} + + public static Meter createMeter(Metrics metrics, String groupName, String baseName, String descriptiveName) { + return new Meter(new WindowedCount(), + metrics.metricName(baseName + "-rate", groupName, + String.format("The number of %s per second", descriptiveName)), + metrics.metricName(baseName + "-total", groupName, + String.format("The total number of %s", descriptiveName))); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java new file mode 100644 index 0000000000000..43e2b0740bc16 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java @@ -0,0 +1,31 @@ +package org.apache.kafka.clients.consumer.internals.metrics; + +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Max; + +public class HeartbeatMetrics extends AbstractConsumerMetrics { + + private final Metrics metrics; + public final Sensor heartbeatSensor; + + public HeartbeatMetrics(Metrics metrics) { + super(); + this.metrics = metrics; + heartbeatSensor = metrics.sensor("heartbeat-latency"); + heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", + groupMetricsName, + "The max time taken to receive a response to a heartbeat request"), new Max()); + heartbeatSensor.add(createMeter(metrics, groupMetricsName, "heartbeat", "heartbeats")); + + } + + public void registerMeasurable(String name, + String description, + Measurable measurable) { + metrics.addMetric( + metrics.metricName(name, groupMetricsName, description), + measurable); + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java index f6fb2a8928631..1e4991131f453 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java @@ -59,6 +59,7 @@ import org.apache.kafka.common.errors.RetriableException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.JoinGroupRequest; @@ -141,6 +142,7 @@ public class AsyncKafkaConsumerTest { private final ApplicationEventHandler applicationEventHandler = mock(ApplicationEventHandler.class); private final ConsumerMetadata metadata = mock(ConsumerMetadata.class); private final LinkedBlockingQueue backgroundEventQueue = new LinkedBlockingQueue<>(); + long rlb = -1L; @AfterEach public void resetAll() { @@ -1398,6 +1400,23 @@ public void testProcessBackgroundEventsTimesOut() throws Exception { } } + @Test + public void testBS() { + rlb = 1000L; + Measurable lastRebalance = (config, now) -> { + return TimeUnit.SECONDS.convert(now - rlb, TimeUnit.MILLISECONDS); + }; + + System.out.println(lastRebalance.measure(null, System.currentTimeMillis())); // Prints some value + +// Update lastRebalanceEndMs + rlb = 2000L; + + System.out.println(lastRebalance.measure(null, System.currentTimeMillis())); // Still uses the captured value, + // not the updated one + + } + private HashMap mockTopicPartitionOffset() { final TopicPartition t0 = new TopicPartition("t0", 2); final TopicPartition t1 = new TopicPartition("t0", 3); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index 17489eab716c0..bdd3836303ac9 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -217,6 +217,7 @@ public ConsumerTestBuilder(Optional groupInfo, boolean enableA retryBackoffMs, DEFAULT_RETRY_BACKOFF_MAX_MS, gi.heartbeatJitterMs)); + HeartbeatRequestManager heartbeat = spy(new HeartbeatRequestManager( logContext, pollTimer, @@ -225,7 +226,8 @@ public ConsumerTestBuilder(Optional groupInfo, boolean enableA mm, heartbeatState, heartbeatRequestState, - backgroundEventHandler)); + backgroundEventHandler, + metrics)); this.coordinatorRequestManager = Optional.of(coordinator); this.commitRequestManager = Optional.of(commit); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index e44414b0677b9..c6335d2a89400 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -28,6 +28,8 @@ import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseData; +import org.apache.kafka.common.metrics.KafkaMetric; +import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; @@ -62,6 +64,7 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MAX_MS; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MS; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -78,6 +81,7 @@ public class HeartbeatRequestManagerTest { private int heartbeatIntervalMs = DEFAULT_HEARTBEAT_INTERVAL_MS; private int maxPollIntervalMs = DEFAULT_MAX_POLL_INTERVAL_MS; private long retryBackoffMaxMs = DEFAULT_RETRY_BACKOFF_MAX_MS; + private String metricGrpPrefix = "test-consumer"; private static final String DEFAULT_GROUP_ID = "groupId"; private ConsumerTestBuilder testBuilder; @@ -95,6 +99,7 @@ public class HeartbeatRequestManagerTest { private final int memberEpoch = 1; private BackgroundEventHandler backgroundEventHandler; private BlockingQueue backgroundEventQueue; + private Metrics metrics = new Metrics(); @BeforeEach public void setUp() { @@ -671,18 +676,15 @@ private ConsumerConfig config() { return new ConsumerConfig(prop); } - private HeartbeatRequestManager createHeartbeatRequestManager() { - LogContext logContext = new LogContext(); - pollTimer = time.timer(maxPollIntervalMs); - return new HeartbeatRequestManager( - logContext, - pollTimer, - config(), - coordinatorRequestManager, - membershipManager, - heartbeatState, - heartbeatRequestState, - backgroundEventHandler); + private void assertMetricsRecorded() { + assertNotNull(getMetric("heartbeat-response-time-max")); + assertNotNull(getMetric("heartbeat-rate")); + assertNotNull(getMetric("heartbeat-total")); + assertNotNull(getMetric("last-heartbeat-seconds-ago")); + } + + private KafkaMetric getMetric(final String name) { + return metrics.metrics().get(metrics.metricName(name, "consumer-coordinator-metrics")); } private HeartbeatRequestManager createHeartbeatRequestManager( @@ -701,6 +703,7 @@ private HeartbeatRequestManager createHeartbeatRequestManager( membershipManager, heartbeatState, heartbeatRequestState, - backgroundEventHandler); + backgroundEventHandler, + metrics); } } From 112f6cebb89b5341f10614ac4e80337d12c2bd59 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Fri, 19 Jan 2024 10:14:24 -0800 Subject: [PATCH 02/28] clean up --- .../internals/metrics/HeartbeatMetrics.java | 16 ++++++++++++++++ .../internals/HeartbeatRequestManagerTest.java | 3 --- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java index 43e2b0740bc16..912295accc41d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.kafka.clients.consumer.internals.metrics; import org.apache.kafka.common.metrics.Measurable; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index c6335d2a89400..41cf9e39fe69c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -81,13 +81,11 @@ public class HeartbeatRequestManagerTest { private int heartbeatIntervalMs = DEFAULT_HEARTBEAT_INTERVAL_MS; private int maxPollIntervalMs = DEFAULT_MAX_POLL_INTERVAL_MS; private long retryBackoffMaxMs = DEFAULT_RETRY_BACKOFF_MAX_MS; - private String metricGrpPrefix = "test-consumer"; private static final String DEFAULT_GROUP_ID = "groupId"; private ConsumerTestBuilder testBuilder; private Time time; private Timer pollTimer; - private ConsumerConfig config; private CoordinatorRequestManager coordinatorRequestManager; private SubscriptionState subscriptions; private Metadata metadata; @@ -118,7 +116,6 @@ private void setUp(Optional groupInfo) { subscriptions = testBuilder.subscriptions; membershipManager = testBuilder.membershipManager.orElseThrow(IllegalStateException::new); metadata = testBuilder.metadata; - config = testBuilder.config; when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); } From 946c7409fd5478ec547886bee7cb288f20bbf177 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Mon, 22 Jan 2024 14:04:26 -0800 Subject: [PATCH 03/28] remove lastHeartbeatsend and use the requestState var instead eee wip --- .../internals/HeartbeatRequestManager.java | 19 ++++----- .../metrics/AbstractConsumerMetrics.java | 14 ++++++- .../internals/metrics/HeartbeatMetrics.java | 20 +++------ .../HeartbeatRequestManagerTest.java | 41 ++++++++++++++++--- 4 files changed, 63 insertions(+), 31 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 04b342fa9a0a5..6feb4154836d5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -82,11 +82,6 @@ public class HeartbeatRequestManager implements RequestManager { */ private final int maxPollIntervalMs; - /** - * The last time in Ms the heartbeat was sent to the coordinator - */ - private long lastHeartbeatSend = 0L; - /** * CoordinatorRequestManager manages the connection to the group coordinator */ @@ -261,7 +256,6 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final long curr NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(ignoreResponse); heartbeatRequestState.onSendAttempt(currentTimeMs); membershipManager.onHeartbeatRequestSent(); - lastHeartbeatSend = currentTimeMs; return request; } @@ -273,7 +267,10 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i return logResponse(request); else return request.whenComplete((response, exception) -> { + System.out.print("====="); if (response != null) { + System.out.print("ehhh: " + response.requestLatencyMs()); + heartbeatMetrics.heartbeatSensor.record(response.requestLatencyMs()); onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), request.handler().completionTimeMs()); } else { onFailure(exception, request.handler().completionTimeMs()); @@ -433,16 +430,18 @@ private void handleFatalFailure(Throwable error) { } private void registerLastSentHeartbeatMetric(Metrics metrics) { + final long lastHeartbeatSend = heartbeatRequestState.lastSentMs; Measurable lastHeartbeat = (config, now) -> { - if (lastHeartbeatSend == 0L) + if (lastHeartbeatSend < 0L) // if no heartbeat is ever triggered, just return -1. return -1d; else return TimeUnit.SECONDS.convert(now - lastHeartbeatSend, TimeUnit.MILLISECONDS); }; - heartbeatMetrics.registerMeasurable( - "heartbeat-time-max", - "The max time taken to receive a response to a heartbeat request", + heartbeatMetrics.addMetric( + metrics, + "last-heartbeat-seconds-ago", + "The number of seconds since the last coordinator heartbeat was sent", lastHeartbeat); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java index c9cd16b26f8cd..1c380b116b14e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; +import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.stats.Meter; import org.apache.kafka.common.metrics.stats.WindowedCount; @@ -27,12 +28,12 @@ public abstract class AbstractConsumerMetrics { protected String groupMetricsName = CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX; + + public AbstractConsumerMetrics() {} public AbstractConsumerMetrics(Optional grpMetricsPrefix) { grpMetricsPrefix.ifPresent(s -> this.groupMetricsName = s + COORDINATOR_METRICS_SUFFIX); } - public AbstractConsumerMetrics() {} - public static Meter createMeter(Metrics metrics, String groupName, String baseName, String descriptiveName) { return new Meter(new WindowedCount(), metrics.metricName(baseName + "-rate", groupName, @@ -40,4 +41,13 @@ public static Meter createMeter(Metrics metrics, String groupName, String baseNa metrics.metricName(baseName + "-total", groupName, String.format("The total number of %s", descriptiveName))); } + + public void addMetric(Metrics metrics, + String name, + String description, + Measurable measurable) { + metrics.addMetric( + metrics.metricName(name, groupMetricsName, description), + measurable); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java index 912295accc41d..ced8aaca6900b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java @@ -16,32 +16,24 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; -import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Max; public class HeartbeatMetrics extends AbstractConsumerMetrics { - private final Metrics metrics; public final Sensor heartbeatSensor; public HeartbeatMetrics(Metrics metrics) { super(); - this.metrics = metrics; heartbeatSensor = metrics.sensor("heartbeat-latency"); heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", groupMetricsName, - "The max time taken to receive a response to a heartbeat request"), new Max()); - heartbeatSensor.add(createMeter(metrics, groupMetricsName, "heartbeat", "heartbeats")); - - } - - public void registerMeasurable(String name, - String description, - Measurable measurable) { - metrics.addMetric( - metrics.metricName(name, groupMetricsName, description), - measurable); + "The max time taken to receive a response to a heartbeat request"), + new Max()); + heartbeatSensor.add(createMeter(metrics, + groupMetricsName, + "heartbeat", + "heartbeats")); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index 41cf9e39fe69c..1dd5d4e1e6a53 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -64,7 +64,6 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MAX_MS; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MS; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -564,6 +563,41 @@ public void testPollTimerExpiration() { assertHeartbeat(heartbeatRequestManager); } + @Test + public void testHeartbeatMetrics() { + coordinatorRequestManager = mock(CoordinatorRequestManager.class); + membershipManager = mock(MembershipManager.class); + heartbeatState = mock(HeartbeatRequestManager.HeartbeatState.class); + heartbeatRequestState = new HeartbeatRequestManager.HeartbeatRequestState( + new LogContext(), + time, + heartbeatIntervalMs, + retryBackoffMs, + retryBackoffMaxMs, + 0); + backgroundEventHandler = mock(BackgroundEventHandler.class); + + heartbeatRequestManager = createHeartbeatRequestManager( + coordinatorRequestManager, + membershipManager, + heartbeatState, + heartbeatRequestState, + backgroundEventHandler); + when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); + when(membershipManager.shouldSkipHeartbeat()).thenReturn(false); + when(membershipManager.state()).thenReturn(MemberState.STABLE); + time.sleep(maxPollIntervalMs); + NetworkClientDelegate.PollResult pollResult = heartbeatRequestManager.poll(time.milliseconds()); + ClientResponse hr = createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE); + pollResult.unsentRequests.get(0).future().complete(hr); + + assertEquals(1, pollResult.unsentRequests.size()); + //assertEquals(1, getMetric("heartbeat-response-time-max").metricValue()); + assertEquals(0.0, getMetric("heartbeat-rate").metricValue()); + assertEquals(1.0, getMetric("heartbeat-total").metricValue()); + assertEquals(1, getMetric("last-heartbeat-seconds-ago").metricValue()); + } + private void assertHeartbeat(HeartbeatRequestManager hrm) { NetworkClientDelegate.PollResult pollResult = hrm.poll(time.milliseconds()); assertEquals(1, pollResult.unsentRequests.size()); @@ -674,10 +708,7 @@ private ConsumerConfig config() { } private void assertMetricsRecorded() { - assertNotNull(getMetric("heartbeat-response-time-max")); - assertNotNull(getMetric("heartbeat-rate")); - assertNotNull(getMetric("heartbeat-total")); - assertNotNull(getMetric("last-heartbeat-seconds-ago")); + } private KafkaMetric getMetric(final String name) { From 6bd15e84976e211b242fbf84d9da546946a1e349 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Thu, 25 Jan 2024 12:48:09 -0800 Subject: [PATCH 04/28] update test clean up remove bad code --- .../internals/HeartbeatRequestManager.java | 25 ++++---- .../internals/AsyncKafkaConsumerTest.java | 18 ------ .../internals/ConsumerTestBuilder.java | 1 - .../HeartbeatRequestManagerTest.java | 58 ++++++++++--------- 4 files changed, 44 insertions(+), 58 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 6feb4154836d5..fdf312a99beab 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -136,8 +136,7 @@ public HeartbeatRequestManager( this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, retryBackoffMaxMs, maxPollIntervalMs); this.pollTimer = time.timer(maxPollIntervalMs); - this.heartbeatMetrics = new HeartbeatMetrics(metrics); - registerLastSentHeartbeatMetric(metrics); + this.heartbeatMetrics = createHeartbeatMetrics(metrics); } // Visible for testing @@ -159,8 +158,7 @@ public HeartbeatRequestManager( this.membershipManager = membershipManager; this.backgroundEventHandler = backgroundEventHandler; this.pollTimer = timer; - this.heartbeatMetrics = new HeartbeatMetrics(metrics); - registerLastSentHeartbeatMetric(metrics); + this.heartbeatMetrics = createHeartbeatMetrics(metrics); } /** @@ -267,9 +265,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i return logResponse(request); else return request.whenComplete((response, exception) -> { - System.out.print("====="); if (response != null) { - System.out.print("ehhh: " + response.requestLatencyMs()); heartbeatMetrics.heartbeatSensor.record(response.requestLatencyMs()); onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), request.handler().completionTimeMs()); } else { @@ -281,6 +277,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i private NetworkClientDelegate.UnsentRequest logResponse(final NetworkClientDelegate.UnsentRequest request) { return request.whenComplete((response, exception) -> { if (response != null) { + heartbeatMetrics.heartbeatSensor.record(response.requestLatencyMs()); Errors error = Errors.forCode(((ConsumerGroupHeartbeatResponse) response.responseBody()).data().errorCode()); if (error == Errors.NONE) @@ -429,20 +426,22 @@ private void handleFatalFailure(Throwable error) { membershipManager.transitionToFatal(); } - private void registerLastSentHeartbeatMetric(Metrics metrics) { - final long lastHeartbeatSend = heartbeatRequestState.lastSentMs; + private HeartbeatMetrics createHeartbeatMetrics(Metrics metrics) { + HeartbeatMetrics hbm = new HeartbeatMetrics(metrics); Measurable lastHeartbeat = (config, now) -> { + final long lastHeartbeatSend = heartbeatRequestState.lastSentMs; if (lastHeartbeatSend < 0L) // if no heartbeat is ever triggered, just return -1. return -1d; else return TimeUnit.SECONDS.convert(now - lastHeartbeatSend, TimeUnit.MILLISECONDS); }; - heartbeatMetrics.addMetric( - metrics, - "last-heartbeat-seconds-ago", - "The number of seconds since the last coordinator heartbeat was sent", - lastHeartbeat); + hbm.addMetric( + metrics, + "last-heartbeat-seconds-ago", + "The number of seconds since the last coordinator heartbeat was sent", + lastHeartbeat); + return hbm; } /** diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java index 1e4991131f453..7c5f4f999cde5 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java @@ -59,7 +59,6 @@ import org.apache.kafka.common.errors.RetriableException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.WakeupException; -import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.JoinGroupRequest; @@ -1400,23 +1399,6 @@ public void testProcessBackgroundEventsTimesOut() throws Exception { } } - @Test - public void testBS() { - rlb = 1000L; - Measurable lastRebalance = (config, now) -> { - return TimeUnit.SECONDS.convert(now - rlb, TimeUnit.MILLISECONDS); - }; - - System.out.println(lastRebalance.measure(null, System.currentTimeMillis())); // Prints some value - -// Update lastRebalanceEndMs - rlb = 2000L; - - System.out.println(lastRebalance.measure(null, System.currentTimeMillis())); // Still uses the captured value, - // not the updated one - - } - private HashMap mockTopicPartitionOffset() { final TopicPartition t0 = new TopicPartition("t0", 2); final TopicPartition t1 = new TopicPartition("t0", 3); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index bdd3836303ac9..62886c7996631 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -217,7 +217,6 @@ public ConsumerTestBuilder(Optional groupInfo, boolean enableA retryBackoffMs, DEFAULT_RETRY_BACKOFF_MAX_MS, gi.heartbeatJitterMs)); - HeartbeatRequestManager heartbeat = spy(new HeartbeatRequestManager( logContext, pollTimer, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index 1dd5d4e1e6a53..e63112b92f46b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -37,6 +37,7 @@ import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource; @@ -63,6 +64,8 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_REMOTE_ASSIGNOR; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MAX_MS; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MS; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -568,34 +571,41 @@ public void testHeartbeatMetrics() { coordinatorRequestManager = mock(CoordinatorRequestManager.class); membershipManager = mock(MembershipManager.class); heartbeatState = mock(HeartbeatRequestManager.HeartbeatState.class); - heartbeatRequestState = new HeartbeatRequestManager.HeartbeatRequestState( - new LogContext(), - time, - heartbeatIntervalMs, - retryBackoffMs, - retryBackoffMaxMs, - 0); + time = new MockTime(); + metrics = new Metrics(time); + heartbeatRequestState = spy(new HeartbeatRequestManager.HeartbeatRequestState( + new LogContext(), + time, + 0, // This initial interval should be 0 to ensure heartbeat on the clock + retryBackoffMs, + retryBackoffMaxMs, + 0)); backgroundEventHandler = mock(BackgroundEventHandler.class); - heartbeatRequestManager = createHeartbeatRequestManager( - coordinatorRequestManager, - membershipManager, - heartbeatState, - heartbeatRequestState, - backgroundEventHandler); + coordinatorRequestManager, + membershipManager, + heartbeatState, + heartbeatRequestState, + backgroundEventHandler); when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); - when(membershipManager.shouldSkipHeartbeat()).thenReturn(false); when(membershipManager.state()).thenReturn(MemberState.STABLE); - time.sleep(maxPollIntervalMs); NetworkClientDelegate.PollResult pollResult = heartbeatRequestManager.poll(time.milliseconds()); - ClientResponse hr = createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE); - pollResult.unsentRequests.get(0).future().complete(hr); - assertEquals(1, pollResult.unsentRequests.size()); - //assertEquals(1, getMetric("heartbeat-response-time-max").metricValue()); - assertEquals(0.0, getMetric("heartbeat-rate").metricValue()); + pollResult.unsentRequests.get(0).handler().onComplete( + createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE)); + time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(1.0, getMetric("heartbeat-total").metricValue()); - assertEquals(1, getMetric("last-heartbeat-seconds-ago").metricValue()); + assertEquals(heartbeatIntervalMs / 1000d, getMetric("last-heartbeat-seconds-ago").metricValue()); + + pollResult = heartbeatRequestManager.poll(time.milliseconds()); + assertEquals(1, pollResult.unsentRequests.size()); + pollResult.unsentRequests.get(0).handler().onComplete( + createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE)); + + assertEquals(0.06d, (double) getMetric("heartbeat-rate").metricValue(), 0.005d); + assertEquals(2.0, getMetric("heartbeat-total").metricValue()); + time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); + assertEquals(heartbeatIntervalMs / 1000d, getMetric("last-heartbeat-seconds-ago").metricValue()); } private void assertHeartbeat(HeartbeatRequestManager hrm) { @@ -707,12 +717,8 @@ private ConsumerConfig config() { return new ConsumerConfig(prop); } - private void assertMetricsRecorded() { - - } - private KafkaMetric getMetric(final String name) { - return metrics.metrics().get(metrics.metricName(name, "consumer-coordinator-metrics")); + return metrics.metrics().get(metrics.metricName(name, CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX)); } private HeartbeatRequestManager createHeartbeatRequestManager( From 6759da15f54b09df0e1970d01d0fc52485b891b0 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Fri, 26 Jan 2024 11:24:46 -0800 Subject: [PATCH 05/28] clean up clean up test --- .../consumer/internals/ConsumerUtils.java | 2 + .../metrics/AbstractConsumerMetrics.java | 65 +++++++-- .../internals/metrics/HeartbeatMetrics.java | 3 +- .../metrics/RebalanceCallbackMetrics.java | 4 +- .../internals/AsyncKafkaConsumerTest.java | 1 - .../internals/ConsumerMetricsTest.java | 125 ++++++++++++++++++ .../HeartbeatRequestManagerTest.java | 27 ++-- 7 files changed, 196 insertions(+), 31 deletions(-) create mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java index bc7590c38f819..ddb79b034c872 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java @@ -66,6 +66,8 @@ public final class ConsumerUtils { public static final String CONSUMER_JMX_PREFIX = "kafka.consumer"; public static final String CONSUMER_METRIC_GROUP_PREFIX = "consumer"; public static final String COORDINATOR_METRICS_SUFFIX = "-coordinator-metrics"; + public static final String CONSUMER_METRICS_SUFFIX = "-metrics"; + public static final String FETCH_MANAGER_METRICS_SUFFIX = "-fetch-manager-metrics"; /** * A fixed, large enough value will suffice for max. diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java index 1c380b116b14e..56f1a310b3f0d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java @@ -16,30 +16,72 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; +import org.apache.kafka.clients.consumer.internals.ConsumerUtils; import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.stats.Meter; import org.apache.kafka.common.metrics.stats.WindowedCount; +import java.util.Objects; import java.util.Optional; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.FETCH_MANAGER_METRICS_SUFFIX; +/** + * This class encapsulates the creation of metrics for the kafka consumer. When constructing a metrics object, you + * need to provide the suffix of the metric group. These are: + *
  • + *
      Group Coordinator: {@link MetricSuffix#COORDINATOR}
    + *
      Consumer metrics: {@link MetricSuffix#CONSUMER}
    + *
      Fetch manager metrics: {@link MetricSuffix#FETCH_MANAGER}
    + *
    • + * + * If no prefix is provided, we will use the default prefix {@link ConsumerUtils#CONSUMER_METRIC_GROUP_PREFIX}. + */ public abstract class AbstractConsumerMetrics { - protected String groupMetricsName = CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX; + protected final String groupMetricsName; + + public enum MetricSuffix { + COORDINATOR(COORDINATOR_METRICS_SUFFIX), + CONSUMER(CONSUMER_METRICS_SUFFIX), + FETCH_MANAGER(FETCH_MANAGER_METRICS_SUFFIX); + + private final String suffix; + + MetricSuffix(String suffix) { + this.suffix = suffix; + } + + public String getSuffix() { + return suffix; + } + } + + public AbstractConsumerMetrics(MetricSuffix suffix) { + this(Optional.empty(), suffix.getSuffix()); + } - public AbstractConsumerMetrics() {} - public AbstractConsumerMetrics(Optional grpMetricsPrefix) { - grpMetricsPrefix.ifPresent(s -> this.groupMetricsName = s + COORDINATOR_METRICS_SUFFIX); + public AbstractConsumerMetrics(Optional grpMetricsPrefix, String suffix) { + Objects.requireNonNull(suffix, "suffix cannot be null"); + String prefix = grpMetricsPrefix.orElse(CONSUMER_METRIC_GROUP_PREFIX); + groupMetricsName = prefix + suffix; } - public static Meter createMeter(Metrics metrics, String groupName, String baseName, String descriptiveName) { + public String groupMetricsName() { + return groupMetricsName; + } + + public Meter createMeter(Metrics metrics, String baseName, String descriptiveName) { return new Meter(new WindowedCount(), - metrics.metricName(baseName + "-rate", groupName, - String.format("The number of %s per second", descriptiveName)), - metrics.metricName(baseName + "-total", groupName, - String.format("The total number of %s", descriptiveName))); + metrics.metricName(baseName + "-rate", + groupMetricsName, + String.format("The number of %s per second", descriptiveName)), + metrics.metricName(baseName + "-total", + groupMetricsName, + String.format("The total number of %s", descriptiveName))); } public void addMetric(Metrics metrics, @@ -47,7 +89,8 @@ public void addMetric(Metrics metrics, String description, Measurable measurable) { metrics.addMetric( - metrics.metricName(name, groupMetricsName, description), - measurable); + metrics.metricName(name, groupMetricsName, description), + measurable); } } + diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java index ced8aaca6900b..ed9c93947ae5e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java @@ -25,14 +25,13 @@ public class HeartbeatMetrics extends AbstractConsumerMetrics { public final Sensor heartbeatSensor; public HeartbeatMetrics(Metrics metrics) { - super(); + super(MetricSuffix.COORDINATOR); heartbeatSensor = metrics.sensor("heartbeat-latency"); heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", groupMetricsName, "The max time taken to receive a response to a heartbeat request"), new Max()); heartbeatSensor.add(createMeter(metrics, - groupMetricsName, "heartbeat", "heartbeats")); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java index a27581505c517..c5850cd241a6e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java @@ -21,8 +21,6 @@ import org.apache.kafka.common.metrics.stats.Avg; import org.apache.kafka.common.metrics.stats.Max; -import java.util.Optional; - public class RebalanceCallbackMetrics extends AbstractConsumerMetrics { public final Sensor revokeCallbackSensor; public final Sensor assignCallbackSensor; @@ -33,7 +31,7 @@ public RebalanceCallbackMetrics(Metrics metrics) { } public RebalanceCallbackMetrics(Metrics metrics, String grpMetricsPrefix) { - super(Optional.ofNullable(grpMetricsPrefix)); + super(MetricSuffix.COORDINATOR); revokeCallbackSensor = metrics.sensor("partition-revoked-latency"); revokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-avg", groupMetricsName, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java index 7c5f4f999cde5..f6fb2a8928631 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java @@ -141,7 +141,6 @@ public class AsyncKafkaConsumerTest { private final ApplicationEventHandler applicationEventHandler = mock(ApplicationEventHandler.class); private final ConsumerMetadata metadata = mock(ConsumerMetadata.class); private final LinkedBlockingQueue backgroundEventQueue = new LinkedBlockingQueue<>(); - long rlb = -1L; @AfterEach public void resetAll() { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java new file mode 100644 index 0000000000000..9072575bada0e --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.consumer.internals.metrics.AbstractConsumerMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.common.metrics.KafkaMetric; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.Time; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.FETCH_MANAGER_METRICS_SUFFIX; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class ConsumerMetricsTest { + private Time time = new MockTime(); + private Metrics metrics = new Metrics(time); + + @Test + public void testRebalanceCallbackMetrics() { + // MBean: kafka.consumer:type=consumer-coordinator-metrics,client-id={clientId} + RebalanceCallbackMetrics rebalanceCallbackMetrics = new RebalanceCallbackMetrics(metrics); + String groupName = "consumer-coordinator-metrics"; + assertNotNull(getMetric("partition-revoked-latency-avg", groupName)); + assertNotNull(getMetric("partition-revoked-latency-max", groupName)); + assertNotNull(getMetric("partition-assigned-latency-avg", groupName)); + assertNotNull(getMetric("partition-assigned-latency-max", groupName)); + assertNotNull(getMetric("partition-lost-latency-avg", groupName)); + assertNotNull(getMetric("partition-lost-latency-max", groupName)); + + rebalanceCallbackMetrics.assignCallbackSensor.record(100); + rebalanceCallbackMetrics.revokeCallbackSensor.record(101); + rebalanceCallbackMetrics.loseCallbackSensor.record(102); + + assertEquals(101d, getMetric("partition-revoked-latency-avg", groupName).metricValue()); + assertEquals(101d, getMetric("partition-revoked-latency-max", groupName).metricValue()); + assertEquals(100d, getMetric("partition-assigned-latency-avg", groupName).metricValue()); + assertEquals(100d, getMetric("partition-assigned-latency-max", groupName).metricValue()); + assertEquals(102d, getMetric("partition-lost-latency-avg", groupName).metricValue()); + assertEquals(102d, getMetric("partition-lost-latency-max", groupName).metricValue()); + } + + @Test + public void testHeartbeatMetrics() { + // MBean: kafka.consumer:type=consumer-coordinator-metrics,client-id={clientId} + HeartbeatMetrics heartbeatMetrics = new HeartbeatMetrics(metrics); + String groupName = "consumer-coordinator-metrics"; + assertNotNull(getMetric("heartbeat-response-time-max", groupName)); + assertNotNull(getMetric("heartbeat-rate", groupName)); + assertNotNull(getMetric("heartbeat-total", groupName)); + + heartbeatMetrics.heartbeatSensor.record(100); + heartbeatMetrics.heartbeatSensor.record(100); + heartbeatMetrics.heartbeatSensor.record(100); + + assertEquals(100d, getMetric("heartbeat-response-time-max", groupName).metricValue()); + assertEquals(0.1d, (double) getMetric("heartbeat-rate", groupName).metricValue(), 0.01d); + assertEquals(3d, getMetric("heartbeat-total", groupName).metricValue()); + } + + @Test + public void testMetricsGroupName() { + // consumer-coordinator-metrics + NoopCoordinatorMetrics defaultCoordinatorMetrics = new NoopCoordinatorMetrics(Optional.empty()); + assertEquals(CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX, defaultCoordinatorMetrics.groupMetricsName()); + NoopCoordinatorMetrics coordinatorMetrics = new NoopCoordinatorMetrics(Optional.of("customCoordinatorPrefix")); + assertEquals("customCoordinatorPrefix" + COORDINATOR_METRICS_SUFFIX, coordinatorMetrics.groupMetricsName()); + + // consumer-metrics + NoopConsumerMetrics defaultConsumerMetrics = new NoopConsumerMetrics(Optional.empty()); + assertEquals(CONSUMER_METRIC_GROUP_PREFIX + CONSUMER_METRICS_SUFFIX, defaultConsumerMetrics.groupMetricsName()); + NoopConsumerMetrics consumerMetrics = new NoopConsumerMetrics(Optional.of("customConsumerPrefix")); + assertEquals("customConsumerPrefix" + CONSUMER_METRICS_SUFFIX, consumerMetrics.groupMetricsName()); + + // consumer-fetch-manager-metrics + NoopFetchManagerMetrics defaultFetchMetrics = new NoopFetchManagerMetrics(Optional.empty()); + assertEquals(CONSUMER_METRIC_GROUP_PREFIX + FETCH_MANAGER_METRICS_SUFFIX, defaultFetchMetrics.groupMetricsName()); + NoopFetchManagerMetrics fetchMetrics = new NoopFetchManagerMetrics(Optional.of("customFetchPrefix")); + assertEquals("customFetchPrefix" + FETCH_MANAGER_METRICS_SUFFIX, fetchMetrics.groupMetricsName()); + } + + private KafkaMetric getMetric(String metricName, String groupName) { + return metrics.metrics().get(metrics.metricName(metricName, groupName)); + } + + static class NoopCoordinatorMetrics extends AbstractConsumerMetrics { + public NoopCoordinatorMetrics(Optional prefix) { + super(prefix, MetricSuffix.COORDINATOR.getSuffix()); + } + } + + static class NoopConsumerMetrics extends AbstractConsumerMetrics { + public NoopConsumerMetrics(Optional prefix) { + super(prefix, MetricSuffix.CONSUMER.getSuffix()); + } + } + + static class NoopFetchManagerMetrics extends AbstractConsumerMetrics { + public NoopFetchManagerMetrics(Optional prefix) { + super(prefix, MetricSuffix.FETCH_MANAGER.getSuffix()); + } + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index e63112b92f46b..8857c0f6c7b44 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -67,6 +67,7 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -554,8 +555,7 @@ public void testPollTimerExpiration() { when(membershipManager.state()).thenReturn(MemberState.STABLE); time.sleep(maxPollIntervalMs); - NetworkClientDelegate.PollResult pollResult = heartbeatRequestManager.poll(time.milliseconds()); - assertEquals(1, pollResult.unsentRequests.size()); + assertHeartbeat(heartbeatRequestManager, heartbeatIntervalMs); verify(heartbeatState).reset(); verify(heartbeatRequestState).reset(); verify(membershipManager).transitionToStale(); @@ -563,7 +563,7 @@ public void testPollTimerExpiration() { assertNoHeartbeat(heartbeatRequestManager); heartbeatRequestManager.resetPollTimer(time.milliseconds()); assertTrue(pollTimer.notExpired()); - assertHeartbeat(heartbeatRequestManager); + assertHeartbeat(heartbeatRequestManager, heartbeatIntervalMs); } @Test @@ -589,29 +589,28 @@ public void testHeartbeatMetrics() { backgroundEventHandler); when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); when(membershipManager.state()).thenReturn(MemberState.STABLE); - NetworkClientDelegate.PollResult pollResult = heartbeatRequestManager.poll(time.milliseconds()); - assertEquals(1, pollResult.unsentRequests.size()); - pollResult.unsentRequests.get(0).handler().onComplete( - createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE)); + + assertNotNull(getMetric("heartbeat-response-time-max")); + assertNotNull(getMetric("heartbeat-rate")); + assertNotNull(getMetric("heartbeat-total")); + assertNotNull(getMetric("last-heartbeat-seconds-ago")); + + assertHeartbeat(heartbeatRequestManager, 0); time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(1.0, getMetric("heartbeat-total").metricValue()); assertEquals(heartbeatIntervalMs / 1000d, getMetric("last-heartbeat-seconds-ago").metricValue()); - pollResult = heartbeatRequestManager.poll(time.milliseconds()); - assertEquals(1, pollResult.unsentRequests.size()); - pollResult.unsentRequests.get(0).handler().onComplete( - createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE)); - + assertHeartbeat(heartbeatRequestManager, DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(0.06d, (double) getMetric("heartbeat-rate").metricValue(), 0.005d); assertEquals(2.0, getMetric("heartbeat-total").metricValue()); time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(heartbeatIntervalMs / 1000d, getMetric("last-heartbeat-seconds-ago").metricValue()); } - private void assertHeartbeat(HeartbeatRequestManager hrm) { + private void assertHeartbeat(HeartbeatRequestManager hrm, int nextPollMs) { NetworkClientDelegate.PollResult pollResult = hrm.poll(time.milliseconds()); assertEquals(1, pollResult.unsentRequests.size()); - assertEquals(DEFAULT_HEARTBEAT_INTERVAL_MS, pollResult.timeUntilNextPollMs); + assertEquals(nextPollMs, pollResult.timeUntilNextPollMs); pollResult.unsentRequests.get(0).handler().onComplete(createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE)); } From 39c5239236ce1c01dd6ed6d67bf1b1da1a8548cd Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Fri, 26 Jan 2024 12:04:01 -0800 Subject: [PATCH 06/28] allow the manager to accept a group prefix --- .../clients/consumer/internals/HeartbeatRequestManager.java | 6 ++++-- .../kafka/clients/consumer/internals/RequestManagers.java | 2 ++ .../clients/consumer/internals/ConsumerTestBuilder.java | 2 ++ .../consumer/internals/HeartbeatRequestManagerTest.java | 2 ++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index fdf312a99beab..9ab81ccef3230 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -124,6 +124,7 @@ public HeartbeatRequestManager( final SubscriptionState subscriptions, final MembershipManager membershipManager, final BackgroundEventHandler backgroundEventHandler, + final String metricGroupPrefix, final Metrics metrics) { this.coordinatorRequestManager = coordinatorRequestManager; this.logger = logContext.logger(getClass()); @@ -136,7 +137,7 @@ public HeartbeatRequestManager( this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, retryBackoffMaxMs, maxPollIntervalMs); this.pollTimer = time.timer(maxPollIntervalMs); - this.heartbeatMetrics = createHeartbeatMetrics(metrics); + this.heartbeatMetrics = createHeartbeatMetrics(metrics, metricGroupPrefix); } // Visible for testing @@ -149,6 +150,7 @@ public HeartbeatRequestManager( final HeartbeatState heartbeatState, final HeartbeatRequestState heartbeatRequestState, final BackgroundEventHandler backgroundEventHandler, + final String metricGroupPrefix, final Metrics metrics) { this.logger = logContext.logger(this.getClass()); this.maxPollIntervalMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); @@ -158,7 +160,7 @@ public HeartbeatRequestManager( this.membershipManager = membershipManager; this.backgroundEventHandler = backgroundEventHandler; this.pollTimer = timer; - this.heartbeatMetrics = createHeartbeatMetrics(metrics); + this.heartbeatMetrics = createHeartbeatMetrics(metrics, metricGroupPrefix); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index 7b3e75be0182c..b98f82d738e82 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -35,6 +35,7 @@ import java.util.function.Supplier; import static java.util.Objects.requireNonNull; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.common.utils.Utils.closeQuietly; /** @@ -191,6 +192,7 @@ protected RequestManagers create() { subscriptions, membershipManager, backgroundEventHandler, + CONSUMER_METRIC_GROUP_PREFIX, metrics); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index 62886c7996631..b747945b73a71 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -48,6 +48,7 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_INSTANCE_ID_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; @@ -226,6 +227,7 @@ public ConsumerTestBuilder(Optional groupInfo, boolean enableA heartbeatState, heartbeatRequestState, backgroundEventHandler, + CONSUMER_METRIC_GROUP_PREFIX, metrics)); this.coordinatorRequestManager = Optional.of(coordinator); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index 8857c0f6c7b44..41a31a7eb18f7 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -85,6 +85,7 @@ public class HeartbeatRequestManagerTest { private int maxPollIntervalMs = DEFAULT_MAX_POLL_INTERVAL_MS; private long retryBackoffMaxMs = DEFAULT_RETRY_BACKOFF_MAX_MS; private static final String DEFAULT_GROUP_ID = "groupId"; + private String metricGroupPrefix = CONSUMER_METRIC_GROUP_PREFIX; private ConsumerTestBuilder testBuilder; private Time time; @@ -737,6 +738,7 @@ private HeartbeatRequestManager createHeartbeatRequestManager( heartbeatState, heartbeatRequestState, backgroundEventHandler, + metricGroupPrefix, metrics); } } From c55dae2f3588ae76f265edace71fccce88964963 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Fri, 26 Jan 2024 14:31:20 -0800 Subject: [PATCH 07/28] clean up --- .../internals/HeartbeatRequestManager.java | 2 +- .../metrics/AbstractConsumerMetrics.java | 60 ++++++++++--------- .../internals/metrics/HeartbeatMetrics.java | 16 +++-- .../metrics/RebalanceCallbackMetrics.java | 14 ++--- .../internals/ConsumerMetricsTest.java | 41 ++++--------- .../HeartbeatRequestManagerTest.java | 2 +- 6 files changed, 65 insertions(+), 70 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 9ab81ccef3230..267b2c465d1b8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -428,7 +428,7 @@ private void handleFatalFailure(Throwable error) { membershipManager.transitionToFatal(); } - private HeartbeatMetrics createHeartbeatMetrics(Metrics metrics) { + private HeartbeatMetrics createHeartbeatMetrics(Metrics metrics, String metricGroupPrefix) { HeartbeatMetrics hbm = new HeartbeatMetrics(metrics); Measurable lastHeartbeat = (config, now) -> { final long lastHeartbeatSend = heartbeatRequestState.lastSentMs; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java index 56f1a310b3f0d..950fdf566cb0d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java @@ -22,74 +22,80 @@ import org.apache.kafka.common.metrics.stats.Meter; import org.apache.kafka.common.metrics.stats.WindowedCount; -import java.util.Objects; -import java.util.Optional; - import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.FETCH_MANAGER_METRICS_SUFFIX; /** - * This class encapsulates the creation of metrics for the kafka consumer. When constructing a metrics object, you - * need to provide the suffix of the metric group. These are: - *
  • - *
      Group Coordinator: {@link MetricSuffix#COORDINATOR}
    - *
      Consumer metrics: {@link MetricSuffix#CONSUMER}
    - *
      Fetch manager metrics: {@link MetricSuffix#FETCH_MANAGER}
    - *
    • + * Base class for different consumer metrics to extend. This class helps to construct the logical group name from the + * given prefix and suffix, and provides a few common utilities. + * + *

    + * Prefix can be provided by the user and is "consumer-" by default (see {@link ConsumerUtils#CONSUMER_METRIC_GROUP_PREFIX}). + *

    * - * If no prefix is provided, we will use the default prefix {@link ConsumerUtils#CONSUMER_METRIC_GROUP_PREFIX}. + *

    + * The suffix can be one of the following: + *

      + *
    • -coordinator-metrics: {@link MetricGroupSuffix#COORDINATOR}
      • + *
    • -metrics: {@link MetricGroupSuffix#CONSUMER}
      • + *
    • -fetch-manager-metrics: {@link MetricGroupSuffix#FETCH_MANAGER}
      • + *
    + *

    */ public abstract class AbstractConsumerMetrics { - protected final String groupMetricsName; + protected final String metricGroupName; - public enum MetricSuffix { + public enum MetricGroupSuffix { COORDINATOR(COORDINATOR_METRICS_SUFFIX), CONSUMER(CONSUMER_METRICS_SUFFIX), FETCH_MANAGER(FETCH_MANAGER_METRICS_SUFFIX); private final String suffix; - MetricSuffix(String suffix) { + MetricGroupSuffix(String suffix) { this.suffix = suffix; } - public String getSuffix() { + @Override + public String toString() { return suffix; } } - public AbstractConsumerMetrics(MetricSuffix suffix) { - this(Optional.empty(), suffix.getSuffix()); + public AbstractConsumerMetrics(MetricGroupSuffix suffix) { + this(CONSUMER_METRIC_GROUP_PREFIX, suffix.toString()); } - public AbstractConsumerMetrics(Optional grpMetricsPrefix, String suffix) { - Objects.requireNonNull(suffix, "suffix cannot be null"); - String prefix = grpMetricsPrefix.orElse(CONSUMER_METRIC_GROUP_PREFIX); - groupMetricsName = prefix + suffix; + public AbstractConsumerMetrics(String prefix, String suffix) { + if (suffix == null) + throw new IllegalArgumentException("metric group suffix cannot be null"); + if (prefix == null) + throw new IllegalArgumentException("metric group prefix cannot be null"); + metricGroupName = prefix + suffix; } - public String groupMetricsName() { - return groupMetricsName; + public String metricGroupName() { + return metricGroupName; } public Meter createMeter(Metrics metrics, String baseName, String descriptiveName) { return new Meter(new WindowedCount(), metrics.metricName(baseName + "-rate", - groupMetricsName, + metricGroupName, String.format("The number of %s per second", descriptiveName)), metrics.metricName(baseName + "-total", - groupMetricsName, + metricGroupName, String.format("The total number of %s", descriptiveName))); } public void addMetric(Metrics metrics, - String name, + String metricName, String description, Measurable measurable) { metrics.addMetric( - metrics.metricName(name, groupMetricsName, description), + metrics.metricName(metricName, metricGroupName, description), measurable); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java index ed9c93947ae5e..4ec2899188cf2 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java @@ -20,19 +20,25 @@ import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Max; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; + public class HeartbeatMetrics extends AbstractConsumerMetrics { public final Sensor heartbeatSensor; public HeartbeatMetrics(Metrics metrics) { - super(MetricSuffix.COORDINATOR); + this(metrics, CONSUMER_METRIC_GROUP_PREFIX); + } + + public HeartbeatMetrics(Metrics metrics, String prefix) { + super(MetricGroupSuffix.COORDINATOR); heartbeatSensor = metrics.sensor("heartbeat-latency"); heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", - groupMetricsName, + metricGroupName, "The max time taken to receive a response to a heartbeat request"), - new Max()); + new Max()); heartbeatSensor.add(createMeter(metrics, - "heartbeat", - "heartbeats")); + "heartbeat", + "heartbeats")); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java index c5850cd241a6e..b0c1bad324663 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java @@ -31,29 +31,29 @@ public RebalanceCallbackMetrics(Metrics metrics) { } public RebalanceCallbackMetrics(Metrics metrics, String grpMetricsPrefix) { - super(MetricSuffix.COORDINATOR); + super(MetricGroupSuffix.COORDINATOR); revokeCallbackSensor = metrics.sensor("partition-revoked-latency"); revokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-avg", - groupMetricsName, + metricGroupName, "The average time taken for a partition-revoked rebalance listener callback"), new Avg()); revokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-max", - groupMetricsName, + metricGroupName, "The max time taken for a partition-revoked rebalance listener callback"), new Max()); assignCallbackSensor = metrics.sensor("partition-assigned-latency"); assignCallbackSensor.add(metrics.metricName("partition-assigned-latency-avg", - groupMetricsName, + metricGroupName, "The average time taken for a partition-assigned rebalance listener callback"), new Avg()); assignCallbackSensor.add(metrics.metricName("partition-assigned-latency-max", - groupMetricsName, + metricGroupName, "The max time taken for a partition-assigned rebalance listener callback"), new Max()); loseCallbackSensor = metrics.sensor("partition-lost-latency"); loseCallbackSensor.add(metrics.metricName("partition-lost-latency-avg", - groupMetricsName, + metricGroupName, "The average time taken for a partition-lost rebalance listener callback"), new Avg()); loseCallbackSensor.add(metrics.metricName("partition-lost-latency-max", - groupMetricsName, + metricGroupName, "The max time taken for a partition-lost rebalance listener callback"), new Max()); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java index 9072575bada0e..343e74ce40965 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java @@ -25,14 +25,12 @@ import org.apache.kafka.common.utils.Time; import org.junit.jupiter.api.Test; -import java.util.Optional; - import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.FETCH_MANAGER_METRICS_SUFFIX; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; public class ConsumerMetricsTest { private Time time = new MockTime(); @@ -83,43 +81,28 @@ public void testHeartbeatMetrics() { @Test public void testMetricsGroupName() { // consumer-coordinator-metrics - NoopCoordinatorMetrics defaultCoordinatorMetrics = new NoopCoordinatorMetrics(Optional.empty()); - assertEquals(CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX, defaultCoordinatorMetrics.groupMetricsName()); - NoopCoordinatorMetrics coordinatorMetrics = new NoopCoordinatorMetrics(Optional.of("customCoordinatorPrefix")); - assertEquals("customCoordinatorPrefix" + COORDINATOR_METRICS_SUFFIX, coordinatorMetrics.groupMetricsName()); + NoopConsumerMetrics coordinatorMetrics = new NoopConsumerMetrics("customCoordinatorPrefix", AbstractConsumerMetrics.MetricGroupSuffix.COORDINATOR); + assertEquals("customCoordinatorPrefix" + COORDINATOR_METRICS_SUFFIX, coordinatorMetrics.metricGroupName()); // consumer-metrics - NoopConsumerMetrics defaultConsumerMetrics = new NoopConsumerMetrics(Optional.empty()); - assertEquals(CONSUMER_METRIC_GROUP_PREFIX + CONSUMER_METRICS_SUFFIX, defaultConsumerMetrics.groupMetricsName()); - NoopConsumerMetrics consumerMetrics = new NoopConsumerMetrics(Optional.of("customConsumerPrefix")); - assertEquals("customConsumerPrefix" + CONSUMER_METRICS_SUFFIX, consumerMetrics.groupMetricsName()); + NoopConsumerMetrics consumerMetrics = new NoopConsumerMetrics("customConsumerPrefix", AbstractConsumerMetrics.MetricGroupSuffix.CONSUMER); + assertEquals("customConsumerPrefix" + CONSUMER_METRICS_SUFFIX, consumerMetrics.metricGroupName()); // consumer-fetch-manager-metrics - NoopFetchManagerMetrics defaultFetchMetrics = new NoopFetchManagerMetrics(Optional.empty()); - assertEquals(CONSUMER_METRIC_GROUP_PREFIX + FETCH_MANAGER_METRICS_SUFFIX, defaultFetchMetrics.groupMetricsName()); - NoopFetchManagerMetrics fetchMetrics = new NoopFetchManagerMetrics(Optional.of("customFetchPrefix")); - assertEquals("customFetchPrefix" + FETCH_MANAGER_METRICS_SUFFIX, fetchMetrics.groupMetricsName()); + NoopConsumerMetrics fetchMetrics = new NoopConsumerMetrics("customFetchPrefix", AbstractConsumerMetrics.MetricGroupSuffix.FETCH_MANAGER); + assertEquals("customFetchPrefix" + FETCH_MANAGER_METRICS_SUFFIX, fetchMetrics.metricGroupName()); + + assertThrows(IllegalArgumentException.class, () -> new NoopConsumerMetrics(null, + AbstractConsumerMetrics.MetricGroupSuffix.CONSUMER)); } private KafkaMetric getMetric(String metricName, String groupName) { return metrics.metrics().get(metrics.metricName(metricName, groupName)); } - static class NoopCoordinatorMetrics extends AbstractConsumerMetrics { - public NoopCoordinatorMetrics(Optional prefix) { - super(prefix, MetricSuffix.COORDINATOR.getSuffix()); - } - } - static class NoopConsumerMetrics extends AbstractConsumerMetrics { - public NoopConsumerMetrics(Optional prefix) { - super(prefix, MetricSuffix.CONSUMER.getSuffix()); - } - } - - static class NoopFetchManagerMetrics extends AbstractConsumerMetrics { - public NoopFetchManagerMetrics(Optional prefix) { - super(prefix, MetricSuffix.FETCH_MANAGER.getSuffix()); + public NoopConsumerMetrics(String prefix, MetricGroupSuffix suffix) { + super(prefix, suffix.toString()); } } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index 41a31a7eb18f7..68822970c17eb 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -718,7 +718,7 @@ private ConsumerConfig config() { } private KafkaMetric getMetric(final String name) { - return metrics.metrics().get(metrics.metricName(name, CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX)); + return metrics.metrics().get(metrics.metricName(name, metricGroupPrefix + COORDINATOR_METRICS_SUFFIX)); } private HeartbeatRequestManager createHeartbeatRequestManager( From df8ac287e194315bd031c3e42c195067d6f91179 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Fri, 26 Jan 2024 15:39:38 -0800 Subject: [PATCH 08/28] Update AbstractConsumerMetrics.java --- .../consumer/internals/metrics/AbstractConsumerMetrics.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java index 950fdf566cb0d..3e27167ddc286 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; -import org.apache.kafka.clients.consumer.internals.ConsumerUtils; import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.stats.Meter; @@ -32,10 +31,6 @@ * given prefix and suffix, and provides a few common utilities. * *

    - * Prefix can be provided by the user and is "consumer-" by default (see {@link ConsumerUtils#CONSUMER_METRIC_GROUP_PREFIX}). - *

    - * - *

    * The suffix can be one of the following: *

      *
    • -coordinator-metrics: {@link MetricGroupSuffix#COORDINATOR}
      • From 61519640aba71332a512daa4a28bb86642e8ab50 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Sun, 28 Jan 2024 10:08:20 -0800 Subject: [PATCH 09/28] Reconsolidate metrics into consumer coordinator metrics clean up --- .../kafka/clients/consumer/KafkaConsumer.java | 2 +- .../internals/AsyncKafkaConsumer.java | 7 +- .../internals/CommitRequestManager.java | 33 +-------- .../internals/ConsumerCoordinator.java | 4 +- .../consumer/internals/ConsumerDelegate.java | 1 + .../ConsumerRebalanceListenerInvoker.java | 12 +-- .../consumer/internals/ConsumerUtils.java | 1 - .../internals/HeartbeatRequestManager.java | 35 ++------- .../internals/LegacyKafkaConsumer.java | 1 + .../consumer/internals/RequestManagers.java | 2 - ...va => AbstractConsumerMetricsManager.java} | 19 ++--- .../internals/metrics/HeartbeatMetrics.java | 44 ----------- .../metrics/HeartbeatMetricsManager.java | 69 +++++++++++++++++ .../{ => metrics}/KafkaConsumerMetrics.java | 10 +-- .../metrics/OffsetCommitMetricsManager.java | 50 +++++++++++++ .../metrics/RebalanceCallbackMetrics.java | 59 --------------- .../RebalanceCallbackMetricsManager.java | 74 +++++++++++++++++++ .../internals/CommitRequestManagerTest.java | 1 - ...t.java => ConsumerMetricsManagerTest.java} | 44 +++++------ .../internals/ConsumerTestBuilder.java | 6 +- .../HeartbeatRequestManagerTest.java | 6 +- .../internals/KafkaConsumerMetricsTest.java | 1 + .../internals/MembershipManagerImplTest.java | 4 +- 23 files changed, 258 insertions(+), 227 deletions(-) rename clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/{AbstractConsumerMetrics.java => AbstractConsumerMetricsManager.java} (81%) delete mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java rename clients/src/main/java/org/apache/kafka/clients/consumer/internals/{ => metrics}/KafkaConsumerMetrics.java (93%) create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java delete mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java rename clients/src/test/java/org/apache/kafka/clients/consumer/internals/{ConsumerMetricsTest.java => ConsumerMetricsManagerTest.java} (75%) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index 82e1f8b93da3c..bb75a40657311 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -20,7 +20,7 @@ import org.apache.kafka.clients.consumer.internals.ConsumerDelegate; import org.apache.kafka.clients.consumer.internals.ConsumerDelegateCreator; import org.apache.kafka.clients.consumer.internals.ConsumerMetadata; -import org.apache.kafka.clients.consumer.internals.KafkaConsumerMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.KafkaConsumerMetrics; import org.apache.kafka.clients.consumer.internals.SubscriptionState; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java index c6b9efa23aef5..9198039a85a44 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java @@ -60,7 +60,8 @@ import org.apache.kafka.clients.consumer.internals.events.TopicMetadataApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.UnsubscribeApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent; -import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.KafkaConsumerMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.KafkaException; @@ -395,7 +396,7 @@ private void process(final ConsumerRebalanceListenerCallbackNeededEvent event) { logContext, subscriptions, time, - new RebalanceCallbackMetrics(metrics) + new RebalanceCallbackMetricsManager(metrics) ); this.backgroundEventProcessor = new BackgroundEventProcessor( logContext, @@ -538,7 +539,7 @@ private void process(final ConsumerRebalanceListenerCallbackNeededEvent event) { logContext, subscriptions, time, - new RebalanceCallbackMetrics(metrics) + new RebalanceCallbackMetricsManager(metrics, CONSUMER_METRIC_GROUP_PREFIX) ); ApiVersions apiVersions = new ApiVersions(); Supplier networkClientDelegateSupplier = () -> new NetworkClientDelegate( diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java index 4ab1084eaa479..286bc9d59b9bf 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.RetriableCommitFailedException; +import org.apache.kafka.clients.consumer.internals.metrics.OffsetCommitMetricsManager; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.DisconnectException; @@ -31,11 +32,6 @@ import org.apache.kafka.common.message.OffsetCommitRequestData; import org.apache.kafka.common.message.OffsetCommitResponseData; import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Avg; -import org.apache.kafka.common.metrics.stats.Max; -import org.apache.kafka.common.metrics.stats.Meter; -import org.apache.kafka.common.metrics.stats.WindowedCount; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.requests.OffsetCommitRequest; @@ -64,8 +60,6 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED; import static org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult.EMPTY; import static org.apache.kafka.common.protocol.Errors.COORDINATOR_LOAD_IN_PROGRESS; @@ -85,7 +79,7 @@ public class CommitRequestManager implements RequestManager, MemberStateListener private final boolean throwOnFetchStableOffsetUnsupported; final PendingRequests pendingRequests; private boolean closing = false; - private Sensor commitSensor; + private OffsetCommitMetricsManager offsetCommitMetricsManager; /** * Latest member ID and epoch received via the {@link #onMemberEpochUpdated(Optional, Optional)}, @@ -114,7 +108,6 @@ public CommitRequestManager( config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG), config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG), OptionalDouble.empty(), - CONSUMER_METRIC_GROUP_PREFIX, metrics); } @@ -130,7 +123,6 @@ public CommitRequestManager( final long retryBackoffMs, final long retryBackoffMaxMs, final OptionalDouble jitter, - final String metricGroupPrefix, final Metrics metrics) { Objects.requireNonNull(coordinatorRequestManager, "Coordinator is needed upon committing offsets"); this.logContext = logContext; @@ -152,7 +144,7 @@ public CommitRequestManager( this.jitter = jitter; this.throwOnFetchStableOffsetUnsupported = config.getBoolean(THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED); this.memberInfo = new MemberInfo(); - this.commitSensor = addCommitSensor(metrics, metricGroupPrefix); + this.offsetCommitMetricsManager = new OffsetCommitMetricsManager(metrics); } /** @@ -392,23 +384,6 @@ public NetworkClientDelegate.PollResult drainPendingOffsetCommitRequests() { return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, requests); } - private Sensor addCommitSensor(Metrics metrics, String metricGrpPrefix) { - String metricGrpName = metricGrpPrefix + COORDINATOR_METRICS_SUFFIX; - Sensor sensor = metrics.sensor("commit-latency"); - sensor.add(metrics.metricName("commit-latency-avg", - metricGrpName, - "The average time taken for a commit request"), new Avg()); - sensor.add(metrics.metricName("commit-latency-max", - metricGrpName, - "The max time taken for a commit request"), new Max()); - sensor.add(new Meter(new WindowedCount(), - metrics.metricName("commit-rate", metricGrpName, - "The number of commit calls per second"), - metrics.metricName("commit-total", metricGrpName, - "The total number of commit calls"))); - return sensor; - } - private class OffsetCommitRequestState extends RetriableRequestState { private final Map offsets; private final String groupId; @@ -521,7 +496,7 @@ public NetworkClientDelegate.UnsentRequest toUnsentRequest() { } public void onResponse(final ClientResponse response) { - commitSensor.record(response.requestLatencyMs()); + offsetCommitMetricsManager.recordRequestLatency(response.requestLatencyMs()); long currentTimeMs = response.receivedTimeMs(); OffsetCommitResponse commitResponse = (OffsetCommitResponse) response.responseBody(); Set unauthorizedTopics = new HashSet<>(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 398baf7fb079c..5af8633806da8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -29,7 +29,7 @@ import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.clients.consumer.RetriableCommitFailedException; import org.apache.kafka.clients.consumer.internals.Utils.TopicPartitionComparator; -import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Node; @@ -235,7 +235,7 @@ public ConsumerCoordinator(GroupRebalanceConfig rebalanceConfig, logContext, subscriptions, time, - new RebalanceCallbackMetrics(metrics, metricGrpPrefix) + new RebalanceCallbackMetricsManager(metrics, metricGrpPrefix) ); this.metadata.requestUpdate(true); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegate.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegate.java index 612827ebe83df..67e8d991d639f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegate.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegate.java @@ -17,6 +17,7 @@ package org.apache.kafka.clients.consumer.internals; import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.internals.metrics.KafkaConsumerMetrics; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.utils.Timer; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java index ea60436deb1ed..ca68158d4cf3b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java @@ -17,7 +17,7 @@ package org.apache.kafka.clients.consumer.internals; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; -import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.errors.WakeupException; @@ -41,12 +41,12 @@ public class ConsumerRebalanceListenerInvoker { private final Logger log; private final SubscriptionState subscriptions; private final Time time; - private final RebalanceCallbackMetrics metrics; + private final RebalanceCallbackMetricsManager metrics; ConsumerRebalanceListenerInvoker(LogContext logContext, SubscriptionState subscriptions, Time time, - RebalanceCallbackMetrics metrics) { + RebalanceCallbackMetricsManager metrics) { this.log = logContext.logger(getClass()); this.subscriptions = subscriptions; this.time = time; @@ -62,7 +62,7 @@ public Exception invokePartitionsAssigned(final SortedSet assign try { final long startMs = time.milliseconds(); listener.get().onPartitionsAssigned(assignedPartitions); - metrics.assignCallbackSensor.record(time.milliseconds() - startMs); + metrics.recordPartitionAssignLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { @@ -88,7 +88,7 @@ public Exception invokePartitionsRevoked(final SortedSet revoked try { final long startMs = time.milliseconds(); listener.get().onPartitionsRevoked(revokedPartitions); - metrics.revokeCallbackSensor.record(time.milliseconds() - startMs); + metrics.recordPartitionRevokeLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { @@ -114,7 +114,7 @@ public Exception invokePartitionsLost(final SortedSet lostPartit try { final long startMs = time.milliseconds(); listener.get().onPartitionsLost(lostPartitions); - metrics.loseCallbackSensor.record(time.milliseconds() - startMs); + metrics.recordPartitionLostLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java index ddb79b034c872..5a7745b40dd0d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java @@ -67,7 +67,6 @@ public final class ConsumerUtils { public static final String CONSUMER_METRIC_GROUP_PREFIX = "consumer"; public static final String COORDINATOR_METRICS_SUFFIX = "-coordinator-metrics"; public static final String CONSUMER_METRICS_SUFFIX = "-metrics"; - public static final String FETCH_MANAGER_METRICS_SUFFIX = "-fetch-manager-metrics"; /** * A fixed, large enough value will suffice for max. diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 267b2c465d1b8..2e0e63f786e40 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -23,12 +23,11 @@ import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.GroupMetadataUpdateEvent; -import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetricsManager; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.GroupAuthorizationException; import org.apache.kafka.common.errors.RetriableException; import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; -import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; @@ -44,7 +43,6 @@ import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -112,7 +110,7 @@ public class HeartbeatRequestManager implements RequestManager { * sending heartbeat until the next poll. */ private final Timer pollTimer; - private final HeartbeatMetrics heartbeatMetrics; + private final HeartbeatMetricsManager heartbeatMetricsManager; private GroupMetadataUpdateEvent previousGroupMetadataUpdateEvent = null; @@ -124,7 +122,6 @@ public HeartbeatRequestManager( final SubscriptionState subscriptions, final MembershipManager membershipManager, final BackgroundEventHandler backgroundEventHandler, - final String metricGroupPrefix, final Metrics metrics) { this.coordinatorRequestManager = coordinatorRequestManager; this.logger = logContext.logger(getClass()); @@ -137,7 +134,7 @@ public HeartbeatRequestManager( this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, retryBackoffMaxMs, maxPollIntervalMs); this.pollTimer = time.timer(maxPollIntervalMs); - this.heartbeatMetrics = createHeartbeatMetrics(metrics, metricGroupPrefix); + this.heartbeatMetricsManager = new HeartbeatMetricsManager(metrics); } // Visible for testing @@ -150,7 +147,6 @@ public HeartbeatRequestManager( final HeartbeatState heartbeatState, final HeartbeatRequestState heartbeatRequestState, final BackgroundEventHandler backgroundEventHandler, - final String metricGroupPrefix, final Metrics metrics) { this.logger = logContext.logger(this.getClass()); this.maxPollIntervalMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); @@ -160,7 +156,7 @@ public HeartbeatRequestManager( this.membershipManager = membershipManager; this.backgroundEventHandler = backgroundEventHandler; this.pollTimer = timer; - this.heartbeatMetrics = createHeartbeatMetrics(metrics, metricGroupPrefix); + this.heartbeatMetricsManager = new HeartbeatMetricsManager(metrics); } /** @@ -256,6 +252,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final long curr NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(ignoreResponse); heartbeatRequestState.onSendAttempt(currentTimeMs); membershipManager.onHeartbeatRequestSent(); + heartbeatMetricsManager.recordHeartbeatSentMs(currentTimeMs); return request; } @@ -268,7 +265,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i else return request.whenComplete((response, exception) -> { if (response != null) { - heartbeatMetrics.heartbeatSensor.record(response.requestLatencyMs()); + heartbeatMetricsManager.recordRequestLatency(response.requestLatencyMs()); onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), request.handler().completionTimeMs()); } else { onFailure(exception, request.handler().completionTimeMs()); @@ -279,7 +276,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i private NetworkClientDelegate.UnsentRequest logResponse(final NetworkClientDelegate.UnsentRequest request) { return request.whenComplete((response, exception) -> { if (response != null) { - heartbeatMetrics.heartbeatSensor.record(response.requestLatencyMs()); + heartbeatMetricsManager.recordRequestLatency(response.requestLatencyMs()); Errors error = Errors.forCode(((ConsumerGroupHeartbeatResponse) response.responseBody()).data().errorCode()); if (error == Errors.NONE) @@ -428,24 +425,6 @@ private void handleFatalFailure(Throwable error) { membershipManager.transitionToFatal(); } - private HeartbeatMetrics createHeartbeatMetrics(Metrics metrics, String metricGroupPrefix) { - HeartbeatMetrics hbm = new HeartbeatMetrics(metrics); - Measurable lastHeartbeat = (config, now) -> { - final long lastHeartbeatSend = heartbeatRequestState.lastSentMs; - if (lastHeartbeatSend < 0L) - // if no heartbeat is ever triggered, just return -1. - return -1d; - else - return TimeUnit.SECONDS.convert(now - lastHeartbeatSend, TimeUnit.MILLISECONDS); - }; - hbm.addMetric( - metrics, - "last-heartbeat-seconds-ago", - "The number of seconds since the last coordinator heartbeat was sent", - lastHeartbeat); - return hbm; - } - /** * Represents the state of a heartbeat request, including logic for timing, retries, and exponential backoff. The * object extends {@link RequestState} to enable exponential backoff and duplicated request handling. The two fields diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/LegacyKafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/LegacyKafkaConsumer.java index bcc8cb40f2d91..80be0959d69cb 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/LegacyKafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/LegacyKafkaConsumer.java @@ -35,6 +35,7 @@ import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.clients.consumer.internals.metrics.KafkaConsumerMetrics; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.KafkaException; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index b98f82d738e82..7b3e75be0182c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -35,7 +35,6 @@ import java.util.function.Supplier; import static java.util.Objects.requireNonNull; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.common.utils.Utils.closeQuietly; /** @@ -192,7 +191,6 @@ protected RequestManagers create() { subscriptions, membershipManager, backgroundEventHandler, - CONSUMER_METRIC_GROUP_PREFIX, metrics); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java similarity index 81% rename from clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java rename to clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java index 3e27167ddc286..1eafcfa95bfbc 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java @@ -22,9 +22,7 @@ import org.apache.kafka.common.metrics.stats.WindowedCount; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.FETCH_MANAGER_METRICS_SUFFIX; /** * Base class for different consumer metrics to extend. This class helps to construct the logical group name from the @@ -35,17 +33,16 @@ *
        *
      • -coordinator-metrics: {@link MetricGroupSuffix#COORDINATOR}
        • *
      • -metrics: {@link MetricGroupSuffix#CONSUMER}
        • - *
      • -fetch-manager-metrics: {@link MetricGroupSuffix#FETCH_MANAGER}
        • *
      *

      */ -public abstract class AbstractConsumerMetrics { +public abstract class AbstractConsumerMetricsManager { protected final String metricGroupName; + final Metrics metrics; public enum MetricGroupSuffix { COORDINATOR(COORDINATOR_METRICS_SUFFIX), - CONSUMER(CONSUMER_METRICS_SUFFIX), - FETCH_MANAGER(FETCH_MANAGER_METRICS_SUFFIX); + CONSUMER(CONSUMER_METRICS_SUFFIX); private final String suffix; @@ -59,15 +56,12 @@ public String toString() { } } - public AbstractConsumerMetrics(MetricGroupSuffix suffix) { - this(CONSUMER_METRIC_GROUP_PREFIX, suffix.toString()); - } - - public AbstractConsumerMetrics(String prefix, String suffix) { + public AbstractConsumerMetricsManager(Metrics metrics, String prefix, MetricGroupSuffix suffix) { if (suffix == null) throw new IllegalArgumentException("metric group suffix cannot be null"); if (prefix == null) throw new IllegalArgumentException("metric group prefix cannot be null"); + this.metrics = metrics; metricGroupName = prefix + suffix; } @@ -85,8 +79,7 @@ public Meter createMeter(Metrics metrics, String baseName, String descriptiveNam String.format("The total number of %s", descriptiveName))); } - public void addMetric(Metrics metrics, - String metricName, + public void addMetric(String metricName, String description, Measurable measurable) { metrics.addMetric( diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java deleted file mode 100644 index 4ec2899188cf2..0000000000000 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals.metrics; - -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Max; - -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; - -public class HeartbeatMetrics extends AbstractConsumerMetrics { - - public final Sensor heartbeatSensor; - - public HeartbeatMetrics(Metrics metrics) { - this(metrics, CONSUMER_METRIC_GROUP_PREFIX); - } - - public HeartbeatMetrics(Metrics metrics, String prefix) { - super(MetricGroupSuffix.COORDINATOR); - heartbeatSensor = metrics.sensor("heartbeat-latency"); - heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", - metricGroupName, - "The max time taken to receive a response to a heartbeat request"), - new Max()); - heartbeatSensor.add(createMeter(metrics, - "heartbeat", - "heartbeats")); - } -} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java new file mode 100644 index 0000000000000..708d89ea2d510 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.metrics; + +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Max; + +import java.util.concurrent.TimeUnit; + +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; + +public class HeartbeatMetricsManager extends AbstractConsumerMetricsManager { + + public final Sensor heartbeatSensor; + private long lastHeartbeatMs = -1L; + + public HeartbeatMetricsManager(Metrics metrics) { + this(metrics, CONSUMER_METRIC_GROUP_PREFIX); + } + + public HeartbeatMetricsManager(Metrics metrics, String prefix) { + super(metrics, prefix, MetricGroupSuffix.COORDINATOR); + heartbeatSensor = metrics.sensor("heartbeat-latency"); + heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", + metricGroupName, + "The max time taken to receive a response to a heartbeat request"), + new Max()); + heartbeatSensor.add(createMeter(metrics, + "heartbeat", + "heartbeats")); + + Measurable lastHeartbeat = (config, now) -> { + final long lastHeartbeatSend = lastHeartbeatMs; + if (lastHeartbeatSend < 0L) + // if no heartbeat is ever triggered, just return -1. + return -1d; + else + return TimeUnit.SECONDS.convert(now - lastHeartbeatSend, TimeUnit.MILLISECONDS); + }; + addMetric( + "last-heartbeat-seconds-ago", + "The number of seconds since the last coordinator heartbeat was sent", + lastHeartbeat); + } + + public void recordHeartbeatSentMs(long lastHeartbeatMs) { + this.lastHeartbeatMs = lastHeartbeatMs; + } + + public void recordRequestLatency(long requestLatencyMs) { + heartbeatSensor.record(requestLatencyMs); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java similarity index 93% rename from clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java rename to clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java index 0dc8a33b11bbb..ce32b9d97e02b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.clients.consumer.internals; +package org.apache.kafka.clients.consumer.internals.metrics; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.metrics.Measurable; @@ -26,20 +26,20 @@ import java.util.concurrent.TimeUnit; -public class KafkaConsumerMetrics implements AutoCloseable { +import static org.apache.kafka.clients.consumer.internals.metrics.AbstractConsumerMetricsManager.MetricGroupSuffix.CONSUMER; + +public class KafkaConsumerMetrics extends AbstractConsumerMetricsManager implements AutoCloseable { private final MetricName lastPollMetricName; private final Sensor timeBetweenPollSensor; private final Sensor pollIdleSensor; private final Sensor committedSensor; private final Sensor commitSyncSensor; - private final Metrics metrics; private long lastPollMs; private long pollStartMs; private long timeSinceLastPollMs; public KafkaConsumerMetrics(Metrics metrics, String metricGrpPrefix) { - this.metrics = metrics; - String metricGroupName = metricGrpPrefix + "-metrics"; + super(metrics, metricGrpPrefix, CONSUMER); Measurable lastPoll = (mConfig, now) -> { if (lastPollMs == 0L) // if no poll is ever triggered, just return -1. diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java new file mode 100644 index 0000000000000..7aef981066820 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.metrics; + +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; +import org.apache.kafka.common.metrics.stats.Max; +import org.apache.kafka.common.metrics.stats.Meter; +import org.apache.kafka.common.metrics.stats.WindowedCount; + +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; + +public class OffsetCommitMetricsManager extends AbstractConsumerMetricsManager { + private final Sensor commitSensor; + + public OffsetCommitMetricsManager(Metrics metrics) { + super(metrics, CONSUMER_METRIC_GROUP_PREFIX, MetricGroupSuffix.COORDINATOR); + commitSensor = metrics.sensor("commit-latency"); + commitSensor.add(metrics.metricName("commit-latency-avg", + metricGroupName(), + "The average time taken for a commit request"), new Avg()); + commitSensor.add(metrics.metricName("commit-latency-max", + metricGroupName(), + "The max time taken for a commit request"), new Max()); + commitSensor.add(new Meter(new WindowedCount(), + metrics.metricName("commit-rate", metricGroupName(), + "The number of commit calls per second"), + metrics.metricName("commit-total", metricGroupName(), + "The total number of commit calls"))); + } + + public void recordRequestLatency(long latencyMs) { + this.commitSensor.record(latencyMs); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java deleted file mode 100644 index b0c1bad324663..0000000000000 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals.metrics; - -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Avg; -import org.apache.kafka.common.metrics.stats.Max; - -public class RebalanceCallbackMetrics extends AbstractConsumerMetrics { - public final Sensor revokeCallbackSensor; - public final Sensor assignCallbackSensor; - public final Sensor loseCallbackSensor; - - public RebalanceCallbackMetrics(Metrics metrics) { - this(metrics, null); - } - - public RebalanceCallbackMetrics(Metrics metrics, String grpMetricsPrefix) { - super(MetricGroupSuffix.COORDINATOR); - revokeCallbackSensor = metrics.sensor("partition-revoked-latency"); - revokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-avg", - metricGroupName, - "The average time taken for a partition-revoked rebalance listener callback"), new Avg()); - revokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-max", - metricGroupName, - "The max time taken for a partition-revoked rebalance listener callback"), new Max()); - - assignCallbackSensor = metrics.sensor("partition-assigned-latency"); - assignCallbackSensor.add(metrics.metricName("partition-assigned-latency-avg", - metricGroupName, - "The average time taken for a partition-assigned rebalance listener callback"), new Avg()); - assignCallbackSensor.add(metrics.metricName("partition-assigned-latency-max", - metricGroupName, - "The max time taken for a partition-assigned rebalance listener callback"), new Max()); - - loseCallbackSensor = metrics.sensor("partition-lost-latency"); - loseCallbackSensor.add(metrics.metricName("partition-lost-latency-avg", - metricGroupName, - "The average time taken for a partition-lost rebalance listener callback"), new Avg()); - loseCallbackSensor.add(metrics.metricName("partition-lost-latency-max", - metricGroupName, - "The max time taken for a partition-lost rebalance listener callback"), new Max()); - } -} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java new file mode 100644 index 0000000000000..19e7267805a0a --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.metrics; + +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; +import org.apache.kafka.common.metrics.stats.Max; + +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.metrics.AbstractConsumerMetricsManager.MetricGroupSuffix.COORDINATOR; + +public class RebalanceCallbackMetricsManager extends AbstractConsumerMetricsManager { + private final Sensor partitionRevokeCallbackSensor; + private final Sensor partitionAssignCallbackSensor; + private final Sensor partitionLostCallbackSensor; + + public RebalanceCallbackMetricsManager(Metrics metrics) { + this(metrics, CONSUMER_METRIC_GROUP_PREFIX); + } + + public RebalanceCallbackMetricsManager(Metrics metrics, String grpMetricsPrefix) { + super(metrics, grpMetricsPrefix, COORDINATOR); + partitionRevokeCallbackSensor = metrics.sensor("partition-revoked-latency"); + partitionRevokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-avg", + metricGroupName, + "The average time taken for a partition-revoked rebalance listener callback"), new Avg()); + partitionRevokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-max", + metricGroupName, + "The max time taken for a partition-revoked rebalance listener callback"), new Max()); + + partitionAssignCallbackSensor = metrics.sensor("partition-assigned-latency"); + partitionAssignCallbackSensor.add(metrics.metricName("partition-assigned-latency-avg", + metricGroupName, + "The average time taken for a partition-assigned rebalance listener callback"), new Avg()); + partitionAssignCallbackSensor.add(metrics.metricName("partition-assigned-latency-max", + metricGroupName, + "The max time taken for a partition-assigned rebalance listener callback"), new Max()); + + partitionLostCallbackSensor = metrics.sensor("partition-lost-latency"); + partitionLostCallbackSensor.add(metrics.metricName("partition-lost-latency-avg", + metricGroupName, + "The average time taken for a partition-lost rebalance listener callback"), new Avg()); + partitionLostCallbackSensor.add(metrics.metricName("partition-lost-latency-max", + metricGroupName, + "The max time taken for a partition-lost rebalance listener callback"), new Max()); + } + + public void recordPartitionRevokeLatency(long latencyMs) { + partitionRevokeCallbackSensor.record(latencyMs); + } + + public void recordPartitionAssignLatency(long latencyMs) { + partitionAssignCallbackSensor.record(latencyMs); + } + + public void recordPartitionLostLatency(long latencyMs) { + partitionLostCallbackSensor.record(latencyMs); + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java index 1ac58248c97be..26ca48502f36e 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java @@ -990,7 +990,6 @@ private CommitRequestManager create(final boolean autoCommitEnabled, final long retryBackoffMs, retryBackoffMaxMs, OptionalDouble.of(0), - consumerMetricGroupPrefix, metrics)); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java similarity index 75% rename from clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java rename to clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java index 343e74ce40965..5fad34987eeef 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java @@ -16,9 +16,9 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.consumer.internals.metrics.AbstractConsumerMetrics; -import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetrics; -import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.AbstractConsumerMetricsManager; +import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetricsManager; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; import org.apache.kafka.common.metrics.KafkaMetric; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.utils.MockTime; @@ -27,19 +27,18 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.FETCH_MANAGER_METRICS_SUFFIX; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; -public class ConsumerMetricsTest { +public class ConsumerMetricsManagerTest { private Time time = new MockTime(); private Metrics metrics = new Metrics(time); @Test public void testRebalanceCallbackMetrics() { // MBean: kafka.consumer:type=consumer-coordinator-metrics,client-id={clientId} - RebalanceCallbackMetrics rebalanceCallbackMetrics = new RebalanceCallbackMetrics(metrics); + RebalanceCallbackMetricsManager rebalanceCallbackMetricsManager = new RebalanceCallbackMetricsManager(metrics); String groupName = "consumer-coordinator-metrics"; assertNotNull(getMetric("partition-revoked-latency-avg", groupName)); assertNotNull(getMetric("partition-revoked-latency-max", groupName)); @@ -48,9 +47,9 @@ public void testRebalanceCallbackMetrics() { assertNotNull(getMetric("partition-lost-latency-avg", groupName)); assertNotNull(getMetric("partition-lost-latency-max", groupName)); - rebalanceCallbackMetrics.assignCallbackSensor.record(100); - rebalanceCallbackMetrics.revokeCallbackSensor.record(101); - rebalanceCallbackMetrics.loseCallbackSensor.record(102); + rebalanceCallbackMetricsManager.recordPartitionAssignLatency(100); + rebalanceCallbackMetricsManager.recordPartitionRevokeLatency(101); + rebalanceCallbackMetricsManager.recordPartitionLostLatency(102); assertEquals(101d, getMetric("partition-revoked-latency-avg", groupName).metricValue()); assertEquals(101d, getMetric("partition-revoked-latency-max", groupName).metricValue()); @@ -63,15 +62,15 @@ public void testRebalanceCallbackMetrics() { @Test public void testHeartbeatMetrics() { // MBean: kafka.consumer:type=consumer-coordinator-metrics,client-id={clientId} - HeartbeatMetrics heartbeatMetrics = new HeartbeatMetrics(metrics); + HeartbeatMetricsManager heartbeatMetricsManager = new HeartbeatMetricsManager(metrics); String groupName = "consumer-coordinator-metrics"; assertNotNull(getMetric("heartbeat-response-time-max", groupName)); assertNotNull(getMetric("heartbeat-rate", groupName)); assertNotNull(getMetric("heartbeat-total", groupName)); - heartbeatMetrics.heartbeatSensor.record(100); - heartbeatMetrics.heartbeatSensor.record(100); - heartbeatMetrics.heartbeatSensor.record(100); + heartbeatMetricsManager.heartbeatSensor.record(100); + heartbeatMetricsManager.heartbeatSensor.record(100); + heartbeatMetricsManager.heartbeatSensor.record(100); assertEquals(100d, getMetric("heartbeat-response-time-max", groupName).metricValue()); assertEquals(0.1d, (double) getMetric("heartbeat-rate", groupName).metricValue(), 0.01d); @@ -81,28 +80,25 @@ public void testHeartbeatMetrics() { @Test public void testMetricsGroupName() { // consumer-coordinator-metrics - NoopConsumerMetrics coordinatorMetrics = new NoopConsumerMetrics("customCoordinatorPrefix", AbstractConsumerMetrics.MetricGroupSuffix.COORDINATOR); + NoopConsumerMetrics coordinatorMetrics = new NoopConsumerMetrics(metrics, "customCoordinatorPrefix", AbstractConsumerMetricsManager.MetricGroupSuffix.COORDINATOR); assertEquals("customCoordinatorPrefix" + COORDINATOR_METRICS_SUFFIX, coordinatorMetrics.metricGroupName()); // consumer-metrics - NoopConsumerMetrics consumerMetrics = new NoopConsumerMetrics("customConsumerPrefix", AbstractConsumerMetrics.MetricGroupSuffix.CONSUMER); + NoopConsumerMetrics consumerMetrics = new NoopConsumerMetrics(metrics, "customConsumerPrefix", AbstractConsumerMetricsManager.MetricGroupSuffix.CONSUMER); assertEquals("customConsumerPrefix" + CONSUMER_METRICS_SUFFIX, consumerMetrics.metricGroupName()); - // consumer-fetch-manager-metrics - NoopConsumerMetrics fetchMetrics = new NoopConsumerMetrics("customFetchPrefix", AbstractConsumerMetrics.MetricGroupSuffix.FETCH_MANAGER); - assertEquals("customFetchPrefix" + FETCH_MANAGER_METRICS_SUFFIX, fetchMetrics.metricGroupName()); - - assertThrows(IllegalArgumentException.class, () -> new NoopConsumerMetrics(null, - AbstractConsumerMetrics.MetricGroupSuffix.CONSUMER)); + assertThrows(IllegalArgumentException.class, () -> new NoopConsumerMetrics(metrics, + null, + AbstractConsumerMetricsManager.MetricGroupSuffix.CONSUMER)); } private KafkaMetric getMetric(String metricName, String groupName) { return metrics.metrics().get(metrics.metricName(metricName, groupName)); } - static class NoopConsumerMetrics extends AbstractConsumerMetrics { - public NoopConsumerMetrics(String prefix, MetricGroupSuffix suffix) { - super(prefix, suffix.toString()); + static class NoopConsumerMetrics extends AbstractConsumerMetricsManager { + public NoopConsumerMetrics(Metrics metrics, String prefix, MetricGroupSuffix suffix) { + super(metrics, prefix, suffix); } } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index b747945b73a71..06bcd22bb597f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -25,7 +25,7 @@ import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; -import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.requests.MetadataResponse; @@ -48,7 +48,6 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_INSTANCE_ID_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; @@ -227,7 +226,6 @@ public ConsumerTestBuilder(Optional groupInfo, boolean enableA heartbeatState, heartbeatRequestState, backgroundEventHandler, - CONSUMER_METRIC_GROUP_PREFIX, metrics)); this.coordinatorRequestManager = Optional.of(coordinator); @@ -274,7 +272,7 @@ public ConsumerTestBuilder(Optional groupInfo, boolean enableA logContext, subscriptions, time, - new RebalanceCallbackMetrics(metrics) + new RebalanceCallbackMetricsManager(metrics) ); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index 68822970c17eb..33d54ab66389d 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -57,6 +57,7 @@ import java.util.Optional; import java.util.Properties; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_GROUP_INSTANCE_ID; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_HEARTBEAT_INTERVAL_MS; @@ -599,13 +600,13 @@ public void testHeartbeatMetrics() { assertHeartbeat(heartbeatRequestManager, 0); time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(1.0, getMetric("heartbeat-total").metricValue()); - assertEquals(heartbeatIntervalMs / 1000d, getMetric("last-heartbeat-seconds-ago").metricValue()); + assertEquals((double) TimeUnit.MILLISECONDS.toSeconds(heartbeatIntervalMs), getMetric("last-heartbeat-seconds-ago").metricValue()); assertHeartbeat(heartbeatRequestManager, DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(0.06d, (double) getMetric("heartbeat-rate").metricValue(), 0.005d); assertEquals(2.0, getMetric("heartbeat-total").metricValue()); time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); - assertEquals(heartbeatIntervalMs / 1000d, getMetric("last-heartbeat-seconds-ago").metricValue()); + assertEquals((double) TimeUnit.MILLISECONDS.toSeconds(heartbeatIntervalMs), getMetric("last-heartbeat-seconds-ago").metricValue()); } private void assertHeartbeat(HeartbeatRequestManager hrm, int nextPollMs) { @@ -738,7 +739,6 @@ private HeartbeatRequestManager createHeartbeatRequestManager( heartbeatState, heartbeatRequestState, backgroundEventHandler, - metricGroupPrefix, metrics); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetricsTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetricsTest.java index 087f90b7efa31..c75ee906e535b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetricsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetricsTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.clients.consumer.internals.metrics.KafkaConsumerMetrics; import org.apache.kafka.common.metrics.Metrics; import org.junit.jupiter.api.Test; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java index 7683a521e0d6a..a87b6efe16152 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java @@ -20,7 +20,7 @@ import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.clients.consumer.internals.events.ConsumerRebalanceListenerCallbackCompletedEvent; import org.apache.kafka.clients.consumer.internals.events.ConsumerRebalanceListenerCallbackNeededEvent; -import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.TopicIdPartition; import org.apache.kafka.common.TopicPartition; @@ -1388,7 +1388,7 @@ private ConsumerRebalanceListenerInvoker consumerRebalanceListenerInvoker() { new LogContext(), subscriptionState, new MockTime(1), - new RebalanceCallbackMetrics(new Metrics()) + new RebalanceCallbackMetricsManager(new Metrics()) ); } From a84f83b90500aa2b142b789e173c4ff92e3bccef Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Sun, 28 Jan 2024 17:04:37 -0800 Subject: [PATCH 10/28] clean up name --- .../consumer/internals/CommitRequestManager.java | 6 +++--- .../internals/ConsumerRebalanceListenerInvoker.java | 12 ++++++------ .../consumer/internals/HeartbeatRequestManager.java | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java index 286bc9d59b9bf..dd9c138912360 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java @@ -79,7 +79,7 @@ public class CommitRequestManager implements RequestManager, MemberStateListener private final boolean throwOnFetchStableOffsetUnsupported; final PendingRequests pendingRequests; private boolean closing = false; - private OffsetCommitMetricsManager offsetCommitMetricsManager; + private OffsetCommitMetricsManager metricsManager; /** * Latest member ID and epoch received via the {@link #onMemberEpochUpdated(Optional, Optional)}, @@ -144,7 +144,7 @@ public CommitRequestManager( this.jitter = jitter; this.throwOnFetchStableOffsetUnsupported = config.getBoolean(THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED); this.memberInfo = new MemberInfo(); - this.offsetCommitMetricsManager = new OffsetCommitMetricsManager(metrics); + this.metricsManager = new OffsetCommitMetricsManager(metrics); } /** @@ -496,7 +496,7 @@ public NetworkClientDelegate.UnsentRequest toUnsentRequest() { } public void onResponse(final ClientResponse response) { - offsetCommitMetricsManager.recordRequestLatency(response.requestLatencyMs()); + metricsManager.recordRequestLatency(response.requestLatencyMs()); long currentTimeMs = response.receivedTimeMs(); OffsetCommitResponse commitResponse = (OffsetCommitResponse) response.responseBody(); Set unauthorizedTopics = new HashSet<>(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java index ca68158d4cf3b..93d5c65bee539 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java @@ -41,16 +41,16 @@ public class ConsumerRebalanceListenerInvoker { private final Logger log; private final SubscriptionState subscriptions; private final Time time; - private final RebalanceCallbackMetricsManager metrics; + private final RebalanceCallbackMetricsManager metricsManager; ConsumerRebalanceListenerInvoker(LogContext logContext, SubscriptionState subscriptions, Time time, - RebalanceCallbackMetricsManager metrics) { + RebalanceCallbackMetricsManager metricsManager) { this.log = logContext.logger(getClass()); this.subscriptions = subscriptions; this.time = time; - this.metrics = metrics; + this.metricsManager = metricsManager; } public Exception invokePartitionsAssigned(final SortedSet assignedPartitions) { @@ -62,7 +62,7 @@ public Exception invokePartitionsAssigned(final SortedSet assign try { final long startMs = time.milliseconds(); listener.get().onPartitionsAssigned(assignedPartitions); - metrics.recordPartitionAssignLatency(time.milliseconds() - startMs); + metricsManager.recordPartitionAssignLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { @@ -88,7 +88,7 @@ public Exception invokePartitionsRevoked(final SortedSet revoked try { final long startMs = time.milliseconds(); listener.get().onPartitionsRevoked(revokedPartitions); - metrics.recordPartitionRevokeLatency(time.milliseconds() - startMs); + metricsManager.recordPartitionRevokeLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { @@ -114,7 +114,7 @@ public Exception invokePartitionsLost(final SortedSet lostPartit try { final long startMs = time.milliseconds(); listener.get().onPartitionsLost(lostPartitions); - metrics.recordPartitionLostLatency(time.milliseconds() - startMs); + metricsManager.recordPartitionLostLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 2e0e63f786e40..7afdff75cd2dd 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -110,7 +110,7 @@ public class HeartbeatRequestManager implements RequestManager { * sending heartbeat until the next poll. */ private final Timer pollTimer; - private final HeartbeatMetricsManager heartbeatMetricsManager; + private final HeartbeatMetricsManager metricsManager; private GroupMetadataUpdateEvent previousGroupMetadataUpdateEvent = null; @@ -134,7 +134,7 @@ public HeartbeatRequestManager( this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, retryBackoffMaxMs, maxPollIntervalMs); this.pollTimer = time.timer(maxPollIntervalMs); - this.heartbeatMetricsManager = new HeartbeatMetricsManager(metrics); + this.metricsManager = new HeartbeatMetricsManager(metrics); } // Visible for testing @@ -156,7 +156,7 @@ public HeartbeatRequestManager( this.membershipManager = membershipManager; this.backgroundEventHandler = backgroundEventHandler; this.pollTimer = timer; - this.heartbeatMetricsManager = new HeartbeatMetricsManager(metrics); + this.metricsManager = new HeartbeatMetricsManager(metrics); } /** @@ -252,7 +252,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final long curr NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(ignoreResponse); heartbeatRequestState.onSendAttempt(currentTimeMs); membershipManager.onHeartbeatRequestSent(); - heartbeatMetricsManager.recordHeartbeatSentMs(currentTimeMs); + metricsManager.recordHeartbeatSentMs(currentTimeMs); return request; } @@ -265,7 +265,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i else return request.whenComplete((response, exception) -> { if (response != null) { - heartbeatMetricsManager.recordRequestLatency(response.requestLatencyMs()); + metricsManager.recordRequestLatency(response.requestLatencyMs()); onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), request.handler().completionTimeMs()); } else { onFailure(exception, request.handler().completionTimeMs()); @@ -276,7 +276,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i private NetworkClientDelegate.UnsentRequest logResponse(final NetworkClientDelegate.UnsentRequest request) { return request.whenComplete((response, exception) -> { if (response != null) { - heartbeatMetricsManager.recordRequestLatency(response.requestLatencyMs()); + metricsManager.recordRequestLatency(response.requestLatencyMs()); Errors error = Errors.forCode(((ConsumerGroupHeartbeatResponse) response.responseBody()).data().errorCode()); if (error == Errors.NONE) From 4f3970128e4cec3cbc7562cc738eec8a7aed6d0a Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Wed, 17 Jan 2024 10:41:56 -0800 Subject: [PATCH 11/28] heartbeat metrics --- .../internals/HeartbeatRequestManager.java | 35 +++++++++++++++++-- .../consumer/internals/RequestManagers.java | 3 +- .../metrics/AbstractConsumerMetrics.java | 14 ++++++++ .../internals/metrics/HeartbeatMetrics.java | 31 ++++++++++++++++ .../internals/AsyncKafkaConsumerTest.java | 19 ++++++++++ .../internals/ConsumerTestBuilder.java | 4 ++- .../HeartbeatRequestManagerTest.java | 29 ++++++++------- 7 files changed, 118 insertions(+), 17 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 11846eedeafdc..04b342fa9a0a5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -23,10 +23,13 @@ import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.GroupMetadataUpdateEvent; +import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetrics; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.GroupAuthorizationException; import org.apache.kafka.common.errors.RetriableException; import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; @@ -41,6 +44,7 @@ import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -78,6 +82,11 @@ public class HeartbeatRequestManager implements RequestManager { */ private final int maxPollIntervalMs; + /** + * The last time in Ms the heartbeat was sent to the coordinator + */ + private long lastHeartbeatSend = 0L; + /** * CoordinatorRequestManager manages the connection to the group coordinator */ @@ -108,6 +117,7 @@ public class HeartbeatRequestManager implements RequestManager { * sending heartbeat until the next poll. */ private final Timer pollTimer; + private final HeartbeatMetrics heartbeatMetrics; private GroupMetadataUpdateEvent previousGroupMetadataUpdateEvent = null; @@ -118,7 +128,8 @@ public HeartbeatRequestManager( final CoordinatorRequestManager coordinatorRequestManager, final SubscriptionState subscriptions, final MembershipManager membershipManager, - final BackgroundEventHandler backgroundEventHandler) { + final BackgroundEventHandler backgroundEventHandler, + final Metrics metrics) { this.coordinatorRequestManager = coordinatorRequestManager; this.logger = logContext.logger(getClass()); this.membershipManager = membershipManager; @@ -130,6 +141,8 @@ public HeartbeatRequestManager( this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, retryBackoffMaxMs, maxPollIntervalMs); this.pollTimer = time.timer(maxPollIntervalMs); + this.heartbeatMetrics = new HeartbeatMetrics(metrics); + registerLastSentHeartbeatMetric(metrics); } // Visible for testing @@ -141,7 +154,8 @@ public HeartbeatRequestManager( final MembershipManager membershipManager, final HeartbeatState heartbeatState, final HeartbeatRequestState heartbeatRequestState, - final BackgroundEventHandler backgroundEventHandler) { + final BackgroundEventHandler backgroundEventHandler, + final Metrics metrics) { this.logger = logContext.logger(this.getClass()); this.maxPollIntervalMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); this.coordinatorRequestManager = coordinatorRequestManager; @@ -150,6 +164,8 @@ public HeartbeatRequestManager( this.membershipManager = membershipManager; this.backgroundEventHandler = backgroundEventHandler; this.pollTimer = timer; + this.heartbeatMetrics = new HeartbeatMetrics(metrics); + registerLastSentHeartbeatMetric(metrics); } /** @@ -245,6 +261,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final long curr NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(ignoreResponse); heartbeatRequestState.onSendAttempt(currentTimeMs); membershipManager.onHeartbeatRequestSent(); + lastHeartbeatSend = currentTimeMs; return request; } @@ -415,6 +432,20 @@ private void handleFatalFailure(Throwable error) { membershipManager.transitionToFatal(); } + private void registerLastSentHeartbeatMetric(Metrics metrics) { + Measurable lastHeartbeat = (config, now) -> { + if (lastHeartbeatSend == 0L) + // if no heartbeat is ever triggered, just return -1. + return -1d; + else + return TimeUnit.SECONDS.convert(now - lastHeartbeatSend, TimeUnit.MILLISECONDS); + }; + heartbeatMetrics.registerMeasurable( + "heartbeat-time-max", + "The max time taken to receive a response to a heartbeat request", + lastHeartbeat); + } + /** * Represents the state of a heartbeat request, including logic for timing, retries, and exponential backoff. The * object extends {@link RequestState} to enable exponential backoff and duplicated request handling. The two fields diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index 6dd9e01216343..7b3e75be0182c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -190,7 +190,8 @@ protected RequestManagers create() { coordinator, subscriptions, membershipManager, - backgroundEventHandler); + backgroundEventHandler, + metrics); } return new RequestManagers( diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java index 2273bfd494a3b..c9cd16b26f8cd 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java @@ -16,6 +16,10 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.stats.Meter; +import org.apache.kafka.common.metrics.stats.WindowedCount; + import java.util.Optional; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; @@ -26,4 +30,14 @@ public abstract class AbstractConsumerMetrics { public AbstractConsumerMetrics(Optional grpMetricsPrefix) { grpMetricsPrefix.ifPresent(s -> this.groupMetricsName = s + COORDINATOR_METRICS_SUFFIX); } + + public AbstractConsumerMetrics() {} + + public static Meter createMeter(Metrics metrics, String groupName, String baseName, String descriptiveName) { + return new Meter(new WindowedCount(), + metrics.metricName(baseName + "-rate", groupName, + String.format("The number of %s per second", descriptiveName)), + metrics.metricName(baseName + "-total", groupName, + String.format("The total number of %s", descriptiveName))); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java new file mode 100644 index 0000000000000..43e2b0740bc16 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java @@ -0,0 +1,31 @@ +package org.apache.kafka.clients.consumer.internals.metrics; + +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Max; + +public class HeartbeatMetrics extends AbstractConsumerMetrics { + + private final Metrics metrics; + public final Sensor heartbeatSensor; + + public HeartbeatMetrics(Metrics metrics) { + super(); + this.metrics = metrics; + heartbeatSensor = metrics.sensor("heartbeat-latency"); + heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", + groupMetricsName, + "The max time taken to receive a response to a heartbeat request"), new Max()); + heartbeatSensor.add(createMeter(metrics, groupMetricsName, "heartbeat", "heartbeats")); + + } + + public void registerMeasurable(String name, + String description, + Measurable measurable) { + metrics.addMetric( + metrics.metricName(name, groupMetricsName, description), + measurable); + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java index f6fb2a8928631..1e4991131f453 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java @@ -59,6 +59,7 @@ import org.apache.kafka.common.errors.RetriableException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.JoinGroupRequest; @@ -141,6 +142,7 @@ public class AsyncKafkaConsumerTest { private final ApplicationEventHandler applicationEventHandler = mock(ApplicationEventHandler.class); private final ConsumerMetadata metadata = mock(ConsumerMetadata.class); private final LinkedBlockingQueue backgroundEventQueue = new LinkedBlockingQueue<>(); + long rlb = -1L; @AfterEach public void resetAll() { @@ -1398,6 +1400,23 @@ public void testProcessBackgroundEventsTimesOut() throws Exception { } } + @Test + public void testBS() { + rlb = 1000L; + Measurable lastRebalance = (config, now) -> { + return TimeUnit.SECONDS.convert(now - rlb, TimeUnit.MILLISECONDS); + }; + + System.out.println(lastRebalance.measure(null, System.currentTimeMillis())); // Prints some value + +// Update lastRebalanceEndMs + rlb = 2000L; + + System.out.println(lastRebalance.measure(null, System.currentTimeMillis())); // Still uses the captured value, + // not the updated one + + } + private HashMap mockTopicPartitionOffset() { final TopicPartition t0 = new TopicPartition("t0", 2); final TopicPartition t1 = new TopicPartition("t0", 3); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index 17489eab716c0..bdd3836303ac9 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -217,6 +217,7 @@ public ConsumerTestBuilder(Optional groupInfo, boolean enableA retryBackoffMs, DEFAULT_RETRY_BACKOFF_MAX_MS, gi.heartbeatJitterMs)); + HeartbeatRequestManager heartbeat = spy(new HeartbeatRequestManager( logContext, pollTimer, @@ -225,7 +226,8 @@ public ConsumerTestBuilder(Optional groupInfo, boolean enableA mm, heartbeatState, heartbeatRequestState, - backgroundEventHandler)); + backgroundEventHandler, + metrics)); this.coordinatorRequestManager = Optional.of(coordinator); this.commitRequestManager = Optional.of(commit); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index e44414b0677b9..c6335d2a89400 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -28,6 +28,8 @@ import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseData; +import org.apache.kafka.common.metrics.KafkaMetric; +import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; @@ -62,6 +64,7 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MAX_MS; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MS; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -78,6 +81,7 @@ public class HeartbeatRequestManagerTest { private int heartbeatIntervalMs = DEFAULT_HEARTBEAT_INTERVAL_MS; private int maxPollIntervalMs = DEFAULT_MAX_POLL_INTERVAL_MS; private long retryBackoffMaxMs = DEFAULT_RETRY_BACKOFF_MAX_MS; + private String metricGrpPrefix = "test-consumer"; private static final String DEFAULT_GROUP_ID = "groupId"; private ConsumerTestBuilder testBuilder; @@ -95,6 +99,7 @@ public class HeartbeatRequestManagerTest { private final int memberEpoch = 1; private BackgroundEventHandler backgroundEventHandler; private BlockingQueue backgroundEventQueue; + private Metrics metrics = new Metrics(); @BeforeEach public void setUp() { @@ -671,18 +676,15 @@ private ConsumerConfig config() { return new ConsumerConfig(prop); } - private HeartbeatRequestManager createHeartbeatRequestManager() { - LogContext logContext = new LogContext(); - pollTimer = time.timer(maxPollIntervalMs); - return new HeartbeatRequestManager( - logContext, - pollTimer, - config(), - coordinatorRequestManager, - membershipManager, - heartbeatState, - heartbeatRequestState, - backgroundEventHandler); + private void assertMetricsRecorded() { + assertNotNull(getMetric("heartbeat-response-time-max")); + assertNotNull(getMetric("heartbeat-rate")); + assertNotNull(getMetric("heartbeat-total")); + assertNotNull(getMetric("last-heartbeat-seconds-ago")); + } + + private KafkaMetric getMetric(final String name) { + return metrics.metrics().get(metrics.metricName(name, "consumer-coordinator-metrics")); } private HeartbeatRequestManager createHeartbeatRequestManager( @@ -701,6 +703,7 @@ private HeartbeatRequestManager createHeartbeatRequestManager( membershipManager, heartbeatState, heartbeatRequestState, - backgroundEventHandler); + backgroundEventHandler, + metrics); } } From 99d6fab721927ad87133552f9d04da1128d42ae9 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Fri, 19 Jan 2024 10:14:24 -0800 Subject: [PATCH 12/28] clean up --- .../internals/metrics/HeartbeatMetrics.java | 16 ++++++++++++++++ .../internals/HeartbeatRequestManagerTest.java | 3 --- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java index 43e2b0740bc16..912295accc41d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.kafka.clients.consumer.internals.metrics; import org.apache.kafka.common.metrics.Measurable; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index c6335d2a89400..41cf9e39fe69c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -81,13 +81,11 @@ public class HeartbeatRequestManagerTest { private int heartbeatIntervalMs = DEFAULT_HEARTBEAT_INTERVAL_MS; private int maxPollIntervalMs = DEFAULT_MAX_POLL_INTERVAL_MS; private long retryBackoffMaxMs = DEFAULT_RETRY_BACKOFF_MAX_MS; - private String metricGrpPrefix = "test-consumer"; private static final String DEFAULT_GROUP_ID = "groupId"; private ConsumerTestBuilder testBuilder; private Time time; private Timer pollTimer; - private ConsumerConfig config; private CoordinatorRequestManager coordinatorRequestManager; private SubscriptionState subscriptions; private Metadata metadata; @@ -118,7 +116,6 @@ private void setUp(Optional groupInfo) { subscriptions = testBuilder.subscriptions; membershipManager = testBuilder.membershipManager.orElseThrow(IllegalStateException::new); metadata = testBuilder.metadata; - config = testBuilder.config; when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); } From cf95499e6a7af89b67dd198b4e3e8dd6ad1b7965 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Mon, 22 Jan 2024 14:04:26 -0800 Subject: [PATCH 13/28] remove lastHeartbeatsend and use the requestState var instead eee wip --- .../internals/HeartbeatRequestManager.java | 19 ++++----- .../metrics/AbstractConsumerMetrics.java | 14 ++++++- .../internals/metrics/HeartbeatMetrics.java | 20 +++------ .../HeartbeatRequestManagerTest.java | 41 ++++++++++++++++--- 4 files changed, 63 insertions(+), 31 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 04b342fa9a0a5..6feb4154836d5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -82,11 +82,6 @@ public class HeartbeatRequestManager implements RequestManager { */ private final int maxPollIntervalMs; - /** - * The last time in Ms the heartbeat was sent to the coordinator - */ - private long lastHeartbeatSend = 0L; - /** * CoordinatorRequestManager manages the connection to the group coordinator */ @@ -261,7 +256,6 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final long curr NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(ignoreResponse); heartbeatRequestState.onSendAttempt(currentTimeMs); membershipManager.onHeartbeatRequestSent(); - lastHeartbeatSend = currentTimeMs; return request; } @@ -273,7 +267,10 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i return logResponse(request); else return request.whenComplete((response, exception) -> { + System.out.print("====="); if (response != null) { + System.out.print("ehhh: " + response.requestLatencyMs()); + heartbeatMetrics.heartbeatSensor.record(response.requestLatencyMs()); onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), request.handler().completionTimeMs()); } else { onFailure(exception, request.handler().completionTimeMs()); @@ -433,16 +430,18 @@ private void handleFatalFailure(Throwable error) { } private void registerLastSentHeartbeatMetric(Metrics metrics) { + final long lastHeartbeatSend = heartbeatRequestState.lastSentMs; Measurable lastHeartbeat = (config, now) -> { - if (lastHeartbeatSend == 0L) + if (lastHeartbeatSend < 0L) // if no heartbeat is ever triggered, just return -1. return -1d; else return TimeUnit.SECONDS.convert(now - lastHeartbeatSend, TimeUnit.MILLISECONDS); }; - heartbeatMetrics.registerMeasurable( - "heartbeat-time-max", - "The max time taken to receive a response to a heartbeat request", + heartbeatMetrics.addMetric( + metrics, + "last-heartbeat-seconds-ago", + "The number of seconds since the last coordinator heartbeat was sent", lastHeartbeat); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java index c9cd16b26f8cd..1c380b116b14e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; +import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.stats.Meter; import org.apache.kafka.common.metrics.stats.WindowedCount; @@ -27,12 +28,12 @@ public abstract class AbstractConsumerMetrics { protected String groupMetricsName = CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX; + + public AbstractConsumerMetrics() {} public AbstractConsumerMetrics(Optional grpMetricsPrefix) { grpMetricsPrefix.ifPresent(s -> this.groupMetricsName = s + COORDINATOR_METRICS_SUFFIX); } - public AbstractConsumerMetrics() {} - public static Meter createMeter(Metrics metrics, String groupName, String baseName, String descriptiveName) { return new Meter(new WindowedCount(), metrics.metricName(baseName + "-rate", groupName, @@ -40,4 +41,13 @@ public static Meter createMeter(Metrics metrics, String groupName, String baseNa metrics.metricName(baseName + "-total", groupName, String.format("The total number of %s", descriptiveName))); } + + public void addMetric(Metrics metrics, + String name, + String description, + Measurable measurable) { + metrics.addMetric( + metrics.metricName(name, groupMetricsName, description), + measurable); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java index 912295accc41d..ced8aaca6900b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java @@ -16,32 +16,24 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; -import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Max; public class HeartbeatMetrics extends AbstractConsumerMetrics { - private final Metrics metrics; public final Sensor heartbeatSensor; public HeartbeatMetrics(Metrics metrics) { super(); - this.metrics = metrics; heartbeatSensor = metrics.sensor("heartbeat-latency"); heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", groupMetricsName, - "The max time taken to receive a response to a heartbeat request"), new Max()); - heartbeatSensor.add(createMeter(metrics, groupMetricsName, "heartbeat", "heartbeats")); - - } - - public void registerMeasurable(String name, - String description, - Measurable measurable) { - metrics.addMetric( - metrics.metricName(name, groupMetricsName, description), - measurable); + "The max time taken to receive a response to a heartbeat request"), + new Max()); + heartbeatSensor.add(createMeter(metrics, + groupMetricsName, + "heartbeat", + "heartbeats")); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index 41cf9e39fe69c..1dd5d4e1e6a53 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -64,7 +64,6 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MAX_MS; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MS; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -564,6 +563,41 @@ public void testPollTimerExpiration() { assertHeartbeat(heartbeatRequestManager); } + @Test + public void testHeartbeatMetrics() { + coordinatorRequestManager = mock(CoordinatorRequestManager.class); + membershipManager = mock(MembershipManager.class); + heartbeatState = mock(HeartbeatRequestManager.HeartbeatState.class); + heartbeatRequestState = new HeartbeatRequestManager.HeartbeatRequestState( + new LogContext(), + time, + heartbeatIntervalMs, + retryBackoffMs, + retryBackoffMaxMs, + 0); + backgroundEventHandler = mock(BackgroundEventHandler.class); + + heartbeatRequestManager = createHeartbeatRequestManager( + coordinatorRequestManager, + membershipManager, + heartbeatState, + heartbeatRequestState, + backgroundEventHandler); + when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); + when(membershipManager.shouldSkipHeartbeat()).thenReturn(false); + when(membershipManager.state()).thenReturn(MemberState.STABLE); + time.sleep(maxPollIntervalMs); + NetworkClientDelegate.PollResult pollResult = heartbeatRequestManager.poll(time.milliseconds()); + ClientResponse hr = createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE); + pollResult.unsentRequests.get(0).future().complete(hr); + + assertEquals(1, pollResult.unsentRequests.size()); + //assertEquals(1, getMetric("heartbeat-response-time-max").metricValue()); + assertEquals(0.0, getMetric("heartbeat-rate").metricValue()); + assertEquals(1.0, getMetric("heartbeat-total").metricValue()); + assertEquals(1, getMetric("last-heartbeat-seconds-ago").metricValue()); + } + private void assertHeartbeat(HeartbeatRequestManager hrm) { NetworkClientDelegate.PollResult pollResult = hrm.poll(time.milliseconds()); assertEquals(1, pollResult.unsentRequests.size()); @@ -674,10 +708,7 @@ private ConsumerConfig config() { } private void assertMetricsRecorded() { - assertNotNull(getMetric("heartbeat-response-time-max")); - assertNotNull(getMetric("heartbeat-rate")); - assertNotNull(getMetric("heartbeat-total")); - assertNotNull(getMetric("last-heartbeat-seconds-ago")); + } private KafkaMetric getMetric(final String name) { From 631a755ed1137ca88c87f458cb95ba96b7be48e2 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Thu, 25 Jan 2024 12:48:09 -0800 Subject: [PATCH 14/28] update test clean up remove bad code --- .../internals/HeartbeatRequestManager.java | 25 ++++---- .../internals/AsyncKafkaConsumerTest.java | 18 ------ .../internals/ConsumerTestBuilder.java | 1 - .../HeartbeatRequestManagerTest.java | 58 ++++++++++--------- 4 files changed, 44 insertions(+), 58 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 6feb4154836d5..fdf312a99beab 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -136,8 +136,7 @@ public HeartbeatRequestManager( this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, retryBackoffMaxMs, maxPollIntervalMs); this.pollTimer = time.timer(maxPollIntervalMs); - this.heartbeatMetrics = new HeartbeatMetrics(metrics); - registerLastSentHeartbeatMetric(metrics); + this.heartbeatMetrics = createHeartbeatMetrics(metrics); } // Visible for testing @@ -159,8 +158,7 @@ public HeartbeatRequestManager( this.membershipManager = membershipManager; this.backgroundEventHandler = backgroundEventHandler; this.pollTimer = timer; - this.heartbeatMetrics = new HeartbeatMetrics(metrics); - registerLastSentHeartbeatMetric(metrics); + this.heartbeatMetrics = createHeartbeatMetrics(metrics); } /** @@ -267,9 +265,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i return logResponse(request); else return request.whenComplete((response, exception) -> { - System.out.print("====="); if (response != null) { - System.out.print("ehhh: " + response.requestLatencyMs()); heartbeatMetrics.heartbeatSensor.record(response.requestLatencyMs()); onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), request.handler().completionTimeMs()); } else { @@ -281,6 +277,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i private NetworkClientDelegate.UnsentRequest logResponse(final NetworkClientDelegate.UnsentRequest request) { return request.whenComplete((response, exception) -> { if (response != null) { + heartbeatMetrics.heartbeatSensor.record(response.requestLatencyMs()); Errors error = Errors.forCode(((ConsumerGroupHeartbeatResponse) response.responseBody()).data().errorCode()); if (error == Errors.NONE) @@ -429,20 +426,22 @@ private void handleFatalFailure(Throwable error) { membershipManager.transitionToFatal(); } - private void registerLastSentHeartbeatMetric(Metrics metrics) { - final long lastHeartbeatSend = heartbeatRequestState.lastSentMs; + private HeartbeatMetrics createHeartbeatMetrics(Metrics metrics) { + HeartbeatMetrics hbm = new HeartbeatMetrics(metrics); Measurable lastHeartbeat = (config, now) -> { + final long lastHeartbeatSend = heartbeatRequestState.lastSentMs; if (lastHeartbeatSend < 0L) // if no heartbeat is ever triggered, just return -1. return -1d; else return TimeUnit.SECONDS.convert(now - lastHeartbeatSend, TimeUnit.MILLISECONDS); }; - heartbeatMetrics.addMetric( - metrics, - "last-heartbeat-seconds-ago", - "The number of seconds since the last coordinator heartbeat was sent", - lastHeartbeat); + hbm.addMetric( + metrics, + "last-heartbeat-seconds-ago", + "The number of seconds since the last coordinator heartbeat was sent", + lastHeartbeat); + return hbm; } /** diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java index 1e4991131f453..7c5f4f999cde5 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java @@ -59,7 +59,6 @@ import org.apache.kafka.common.errors.RetriableException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.WakeupException; -import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.JoinGroupRequest; @@ -1400,23 +1399,6 @@ public void testProcessBackgroundEventsTimesOut() throws Exception { } } - @Test - public void testBS() { - rlb = 1000L; - Measurable lastRebalance = (config, now) -> { - return TimeUnit.SECONDS.convert(now - rlb, TimeUnit.MILLISECONDS); - }; - - System.out.println(lastRebalance.measure(null, System.currentTimeMillis())); // Prints some value - -// Update lastRebalanceEndMs - rlb = 2000L; - - System.out.println(lastRebalance.measure(null, System.currentTimeMillis())); // Still uses the captured value, - // not the updated one - - } - private HashMap mockTopicPartitionOffset() { final TopicPartition t0 = new TopicPartition("t0", 2); final TopicPartition t1 = new TopicPartition("t0", 3); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index bdd3836303ac9..62886c7996631 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -217,7 +217,6 @@ public ConsumerTestBuilder(Optional groupInfo, boolean enableA retryBackoffMs, DEFAULT_RETRY_BACKOFF_MAX_MS, gi.heartbeatJitterMs)); - HeartbeatRequestManager heartbeat = spy(new HeartbeatRequestManager( logContext, pollTimer, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index 1dd5d4e1e6a53..e63112b92f46b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -37,6 +37,7 @@ import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource; @@ -63,6 +64,8 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_REMOTE_ASSIGNOR; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MAX_MS; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MS; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -568,34 +571,41 @@ public void testHeartbeatMetrics() { coordinatorRequestManager = mock(CoordinatorRequestManager.class); membershipManager = mock(MembershipManager.class); heartbeatState = mock(HeartbeatRequestManager.HeartbeatState.class); - heartbeatRequestState = new HeartbeatRequestManager.HeartbeatRequestState( - new LogContext(), - time, - heartbeatIntervalMs, - retryBackoffMs, - retryBackoffMaxMs, - 0); + time = new MockTime(); + metrics = new Metrics(time); + heartbeatRequestState = spy(new HeartbeatRequestManager.HeartbeatRequestState( + new LogContext(), + time, + 0, // This initial interval should be 0 to ensure heartbeat on the clock + retryBackoffMs, + retryBackoffMaxMs, + 0)); backgroundEventHandler = mock(BackgroundEventHandler.class); - heartbeatRequestManager = createHeartbeatRequestManager( - coordinatorRequestManager, - membershipManager, - heartbeatState, - heartbeatRequestState, - backgroundEventHandler); + coordinatorRequestManager, + membershipManager, + heartbeatState, + heartbeatRequestState, + backgroundEventHandler); when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); - when(membershipManager.shouldSkipHeartbeat()).thenReturn(false); when(membershipManager.state()).thenReturn(MemberState.STABLE); - time.sleep(maxPollIntervalMs); NetworkClientDelegate.PollResult pollResult = heartbeatRequestManager.poll(time.milliseconds()); - ClientResponse hr = createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE); - pollResult.unsentRequests.get(0).future().complete(hr); - assertEquals(1, pollResult.unsentRequests.size()); - //assertEquals(1, getMetric("heartbeat-response-time-max").metricValue()); - assertEquals(0.0, getMetric("heartbeat-rate").metricValue()); + pollResult.unsentRequests.get(0).handler().onComplete( + createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE)); + time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(1.0, getMetric("heartbeat-total").metricValue()); - assertEquals(1, getMetric("last-heartbeat-seconds-ago").metricValue()); + assertEquals(heartbeatIntervalMs / 1000d, getMetric("last-heartbeat-seconds-ago").metricValue()); + + pollResult = heartbeatRequestManager.poll(time.milliseconds()); + assertEquals(1, pollResult.unsentRequests.size()); + pollResult.unsentRequests.get(0).handler().onComplete( + createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE)); + + assertEquals(0.06d, (double) getMetric("heartbeat-rate").metricValue(), 0.005d); + assertEquals(2.0, getMetric("heartbeat-total").metricValue()); + time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); + assertEquals(heartbeatIntervalMs / 1000d, getMetric("last-heartbeat-seconds-ago").metricValue()); } private void assertHeartbeat(HeartbeatRequestManager hrm) { @@ -707,12 +717,8 @@ private ConsumerConfig config() { return new ConsumerConfig(prop); } - private void assertMetricsRecorded() { - - } - private KafkaMetric getMetric(final String name) { - return metrics.metrics().get(metrics.metricName(name, "consumer-coordinator-metrics")); + return metrics.metrics().get(metrics.metricName(name, CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX)); } private HeartbeatRequestManager createHeartbeatRequestManager( From e96a06e525b9b20e38787acedede126083d2ad07 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Fri, 26 Jan 2024 11:24:46 -0800 Subject: [PATCH 15/28] clean up clean up test --- .../consumer/internals/ConsumerUtils.java | 2 + .../metrics/AbstractConsumerMetrics.java | 65 +++++++-- .../internals/metrics/HeartbeatMetrics.java | 3 +- .../metrics/RebalanceCallbackMetrics.java | 4 +- .../internals/AsyncKafkaConsumerTest.java | 1 - .../internals/ConsumerMetricsTest.java | 125 ++++++++++++++++++ .../HeartbeatRequestManagerTest.java | 27 ++-- 7 files changed, 196 insertions(+), 31 deletions(-) create mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java index bc7590c38f819..ddb79b034c872 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java @@ -66,6 +66,8 @@ public final class ConsumerUtils { public static final String CONSUMER_JMX_PREFIX = "kafka.consumer"; public static final String CONSUMER_METRIC_GROUP_PREFIX = "consumer"; public static final String COORDINATOR_METRICS_SUFFIX = "-coordinator-metrics"; + public static final String CONSUMER_METRICS_SUFFIX = "-metrics"; + public static final String FETCH_MANAGER_METRICS_SUFFIX = "-fetch-manager-metrics"; /** * A fixed, large enough value will suffice for max. diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java index 1c380b116b14e..56f1a310b3f0d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java @@ -16,30 +16,72 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; +import org.apache.kafka.clients.consumer.internals.ConsumerUtils; import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.stats.Meter; import org.apache.kafka.common.metrics.stats.WindowedCount; +import java.util.Objects; import java.util.Optional; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.FETCH_MANAGER_METRICS_SUFFIX; +/** + * This class encapsulates the creation of metrics for the kafka consumer. When constructing a metrics object, you + * need to provide the suffix of the metric group. These are: + *
    • + *
        Group Coordinator: {@link MetricSuffix#COORDINATOR}
      + *
        Consumer metrics: {@link MetricSuffix#CONSUMER}
      + *
        Fetch manager metrics: {@link MetricSuffix#FETCH_MANAGER}
      + *
      • + * + * If no prefix is provided, we will use the default prefix {@link ConsumerUtils#CONSUMER_METRIC_GROUP_PREFIX}. + */ public abstract class AbstractConsumerMetrics { - protected String groupMetricsName = CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX; + protected final String groupMetricsName; + + public enum MetricSuffix { + COORDINATOR(COORDINATOR_METRICS_SUFFIX), + CONSUMER(CONSUMER_METRICS_SUFFIX), + FETCH_MANAGER(FETCH_MANAGER_METRICS_SUFFIX); + + private final String suffix; + + MetricSuffix(String suffix) { + this.suffix = suffix; + } + + public String getSuffix() { + return suffix; + } + } + + public AbstractConsumerMetrics(MetricSuffix suffix) { + this(Optional.empty(), suffix.getSuffix()); + } - public AbstractConsumerMetrics() {} - public AbstractConsumerMetrics(Optional grpMetricsPrefix) { - grpMetricsPrefix.ifPresent(s -> this.groupMetricsName = s + COORDINATOR_METRICS_SUFFIX); + public AbstractConsumerMetrics(Optional grpMetricsPrefix, String suffix) { + Objects.requireNonNull(suffix, "suffix cannot be null"); + String prefix = grpMetricsPrefix.orElse(CONSUMER_METRIC_GROUP_PREFIX); + groupMetricsName = prefix + suffix; } - public static Meter createMeter(Metrics metrics, String groupName, String baseName, String descriptiveName) { + public String groupMetricsName() { + return groupMetricsName; + } + + public Meter createMeter(Metrics metrics, String baseName, String descriptiveName) { return new Meter(new WindowedCount(), - metrics.metricName(baseName + "-rate", groupName, - String.format("The number of %s per second", descriptiveName)), - metrics.metricName(baseName + "-total", groupName, - String.format("The total number of %s", descriptiveName))); + metrics.metricName(baseName + "-rate", + groupMetricsName, + String.format("The number of %s per second", descriptiveName)), + metrics.metricName(baseName + "-total", + groupMetricsName, + String.format("The total number of %s", descriptiveName))); } public void addMetric(Metrics metrics, @@ -47,7 +89,8 @@ public void addMetric(Metrics metrics, String description, Measurable measurable) { metrics.addMetric( - metrics.metricName(name, groupMetricsName, description), - measurable); + metrics.metricName(name, groupMetricsName, description), + measurable); } } + diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java index ced8aaca6900b..ed9c93947ae5e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java @@ -25,14 +25,13 @@ public class HeartbeatMetrics extends AbstractConsumerMetrics { public final Sensor heartbeatSensor; public HeartbeatMetrics(Metrics metrics) { - super(); + super(MetricSuffix.COORDINATOR); heartbeatSensor = metrics.sensor("heartbeat-latency"); heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", groupMetricsName, "The max time taken to receive a response to a heartbeat request"), new Max()); heartbeatSensor.add(createMeter(metrics, - groupMetricsName, "heartbeat", "heartbeats")); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java index a27581505c517..c5850cd241a6e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java @@ -21,8 +21,6 @@ import org.apache.kafka.common.metrics.stats.Avg; import org.apache.kafka.common.metrics.stats.Max; -import java.util.Optional; - public class RebalanceCallbackMetrics extends AbstractConsumerMetrics { public final Sensor revokeCallbackSensor; public final Sensor assignCallbackSensor; @@ -33,7 +31,7 @@ public RebalanceCallbackMetrics(Metrics metrics) { } public RebalanceCallbackMetrics(Metrics metrics, String grpMetricsPrefix) { - super(Optional.ofNullable(grpMetricsPrefix)); + super(MetricSuffix.COORDINATOR); revokeCallbackSensor = metrics.sensor("partition-revoked-latency"); revokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-avg", groupMetricsName, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java index 7c5f4f999cde5..f6fb2a8928631 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java @@ -141,7 +141,6 @@ public class AsyncKafkaConsumerTest { private final ApplicationEventHandler applicationEventHandler = mock(ApplicationEventHandler.class); private final ConsumerMetadata metadata = mock(ConsumerMetadata.class); private final LinkedBlockingQueue backgroundEventQueue = new LinkedBlockingQueue<>(); - long rlb = -1L; @AfterEach public void resetAll() { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java new file mode 100644 index 0000000000000..9072575bada0e --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.consumer.internals.metrics.AbstractConsumerMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.common.metrics.KafkaMetric; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.Time; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.FETCH_MANAGER_METRICS_SUFFIX; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class ConsumerMetricsTest { + private Time time = new MockTime(); + private Metrics metrics = new Metrics(time); + + @Test + public void testRebalanceCallbackMetrics() { + // MBean: kafka.consumer:type=consumer-coordinator-metrics,client-id={clientId} + RebalanceCallbackMetrics rebalanceCallbackMetrics = new RebalanceCallbackMetrics(metrics); + String groupName = "consumer-coordinator-metrics"; + assertNotNull(getMetric("partition-revoked-latency-avg", groupName)); + assertNotNull(getMetric("partition-revoked-latency-max", groupName)); + assertNotNull(getMetric("partition-assigned-latency-avg", groupName)); + assertNotNull(getMetric("partition-assigned-latency-max", groupName)); + assertNotNull(getMetric("partition-lost-latency-avg", groupName)); + assertNotNull(getMetric("partition-lost-latency-max", groupName)); + + rebalanceCallbackMetrics.assignCallbackSensor.record(100); + rebalanceCallbackMetrics.revokeCallbackSensor.record(101); + rebalanceCallbackMetrics.loseCallbackSensor.record(102); + + assertEquals(101d, getMetric("partition-revoked-latency-avg", groupName).metricValue()); + assertEquals(101d, getMetric("partition-revoked-latency-max", groupName).metricValue()); + assertEquals(100d, getMetric("partition-assigned-latency-avg", groupName).metricValue()); + assertEquals(100d, getMetric("partition-assigned-latency-max", groupName).metricValue()); + assertEquals(102d, getMetric("partition-lost-latency-avg", groupName).metricValue()); + assertEquals(102d, getMetric("partition-lost-latency-max", groupName).metricValue()); + } + + @Test + public void testHeartbeatMetrics() { + // MBean: kafka.consumer:type=consumer-coordinator-metrics,client-id={clientId} + HeartbeatMetrics heartbeatMetrics = new HeartbeatMetrics(metrics); + String groupName = "consumer-coordinator-metrics"; + assertNotNull(getMetric("heartbeat-response-time-max", groupName)); + assertNotNull(getMetric("heartbeat-rate", groupName)); + assertNotNull(getMetric("heartbeat-total", groupName)); + + heartbeatMetrics.heartbeatSensor.record(100); + heartbeatMetrics.heartbeatSensor.record(100); + heartbeatMetrics.heartbeatSensor.record(100); + + assertEquals(100d, getMetric("heartbeat-response-time-max", groupName).metricValue()); + assertEquals(0.1d, (double) getMetric("heartbeat-rate", groupName).metricValue(), 0.01d); + assertEquals(3d, getMetric("heartbeat-total", groupName).metricValue()); + } + + @Test + public void testMetricsGroupName() { + // consumer-coordinator-metrics + NoopCoordinatorMetrics defaultCoordinatorMetrics = new NoopCoordinatorMetrics(Optional.empty()); + assertEquals(CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX, defaultCoordinatorMetrics.groupMetricsName()); + NoopCoordinatorMetrics coordinatorMetrics = new NoopCoordinatorMetrics(Optional.of("customCoordinatorPrefix")); + assertEquals("customCoordinatorPrefix" + COORDINATOR_METRICS_SUFFIX, coordinatorMetrics.groupMetricsName()); + + // consumer-metrics + NoopConsumerMetrics defaultConsumerMetrics = new NoopConsumerMetrics(Optional.empty()); + assertEquals(CONSUMER_METRIC_GROUP_PREFIX + CONSUMER_METRICS_SUFFIX, defaultConsumerMetrics.groupMetricsName()); + NoopConsumerMetrics consumerMetrics = new NoopConsumerMetrics(Optional.of("customConsumerPrefix")); + assertEquals("customConsumerPrefix" + CONSUMER_METRICS_SUFFIX, consumerMetrics.groupMetricsName()); + + // consumer-fetch-manager-metrics + NoopFetchManagerMetrics defaultFetchMetrics = new NoopFetchManagerMetrics(Optional.empty()); + assertEquals(CONSUMER_METRIC_GROUP_PREFIX + FETCH_MANAGER_METRICS_SUFFIX, defaultFetchMetrics.groupMetricsName()); + NoopFetchManagerMetrics fetchMetrics = new NoopFetchManagerMetrics(Optional.of("customFetchPrefix")); + assertEquals("customFetchPrefix" + FETCH_MANAGER_METRICS_SUFFIX, fetchMetrics.groupMetricsName()); + } + + private KafkaMetric getMetric(String metricName, String groupName) { + return metrics.metrics().get(metrics.metricName(metricName, groupName)); + } + + static class NoopCoordinatorMetrics extends AbstractConsumerMetrics { + public NoopCoordinatorMetrics(Optional prefix) { + super(prefix, MetricSuffix.COORDINATOR.getSuffix()); + } + } + + static class NoopConsumerMetrics extends AbstractConsumerMetrics { + public NoopConsumerMetrics(Optional prefix) { + super(prefix, MetricSuffix.CONSUMER.getSuffix()); + } + } + + static class NoopFetchManagerMetrics extends AbstractConsumerMetrics { + public NoopFetchManagerMetrics(Optional prefix) { + super(prefix, MetricSuffix.FETCH_MANAGER.getSuffix()); + } + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index e63112b92f46b..8857c0f6c7b44 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -67,6 +67,7 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -554,8 +555,7 @@ public void testPollTimerExpiration() { when(membershipManager.state()).thenReturn(MemberState.STABLE); time.sleep(maxPollIntervalMs); - NetworkClientDelegate.PollResult pollResult = heartbeatRequestManager.poll(time.milliseconds()); - assertEquals(1, pollResult.unsentRequests.size()); + assertHeartbeat(heartbeatRequestManager, heartbeatIntervalMs); verify(heartbeatState).reset(); verify(heartbeatRequestState).reset(); verify(membershipManager).transitionToStale(); @@ -563,7 +563,7 @@ public void testPollTimerExpiration() { assertNoHeartbeat(heartbeatRequestManager); heartbeatRequestManager.resetPollTimer(time.milliseconds()); assertTrue(pollTimer.notExpired()); - assertHeartbeat(heartbeatRequestManager); + assertHeartbeat(heartbeatRequestManager, heartbeatIntervalMs); } @Test @@ -589,29 +589,28 @@ public void testHeartbeatMetrics() { backgroundEventHandler); when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); when(membershipManager.state()).thenReturn(MemberState.STABLE); - NetworkClientDelegate.PollResult pollResult = heartbeatRequestManager.poll(time.milliseconds()); - assertEquals(1, pollResult.unsentRequests.size()); - pollResult.unsentRequests.get(0).handler().onComplete( - createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE)); + + assertNotNull(getMetric("heartbeat-response-time-max")); + assertNotNull(getMetric("heartbeat-rate")); + assertNotNull(getMetric("heartbeat-total")); + assertNotNull(getMetric("last-heartbeat-seconds-ago")); + + assertHeartbeat(heartbeatRequestManager, 0); time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(1.0, getMetric("heartbeat-total").metricValue()); assertEquals(heartbeatIntervalMs / 1000d, getMetric("last-heartbeat-seconds-ago").metricValue()); - pollResult = heartbeatRequestManager.poll(time.milliseconds()); - assertEquals(1, pollResult.unsentRequests.size()); - pollResult.unsentRequests.get(0).handler().onComplete( - createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE)); - + assertHeartbeat(heartbeatRequestManager, DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(0.06d, (double) getMetric("heartbeat-rate").metricValue(), 0.005d); assertEquals(2.0, getMetric("heartbeat-total").metricValue()); time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(heartbeatIntervalMs / 1000d, getMetric("last-heartbeat-seconds-ago").metricValue()); } - private void assertHeartbeat(HeartbeatRequestManager hrm) { + private void assertHeartbeat(HeartbeatRequestManager hrm, int nextPollMs) { NetworkClientDelegate.PollResult pollResult = hrm.poll(time.milliseconds()); assertEquals(1, pollResult.unsentRequests.size()); - assertEquals(DEFAULT_HEARTBEAT_INTERVAL_MS, pollResult.timeUntilNextPollMs); + assertEquals(nextPollMs, pollResult.timeUntilNextPollMs); pollResult.unsentRequests.get(0).handler().onComplete(createHeartbeatResponse(pollResult.unsentRequests.get(0), Errors.NONE)); } From 6043316c25234cb6a0799bf6cc468314d8afdefe Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Fri, 26 Jan 2024 12:04:01 -0800 Subject: [PATCH 16/28] allow the manager to accept a group prefix --- .../clients/consumer/internals/HeartbeatRequestManager.java | 6 ++++-- .../kafka/clients/consumer/internals/RequestManagers.java | 2 ++ .../clients/consumer/internals/ConsumerTestBuilder.java | 2 ++ .../consumer/internals/HeartbeatRequestManagerTest.java | 2 ++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index fdf312a99beab..9ab81ccef3230 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -124,6 +124,7 @@ public HeartbeatRequestManager( final SubscriptionState subscriptions, final MembershipManager membershipManager, final BackgroundEventHandler backgroundEventHandler, + final String metricGroupPrefix, final Metrics metrics) { this.coordinatorRequestManager = coordinatorRequestManager; this.logger = logContext.logger(getClass()); @@ -136,7 +137,7 @@ public HeartbeatRequestManager( this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, retryBackoffMaxMs, maxPollIntervalMs); this.pollTimer = time.timer(maxPollIntervalMs); - this.heartbeatMetrics = createHeartbeatMetrics(metrics); + this.heartbeatMetrics = createHeartbeatMetrics(metrics, metricGroupPrefix); } // Visible for testing @@ -149,6 +150,7 @@ public HeartbeatRequestManager( final HeartbeatState heartbeatState, final HeartbeatRequestState heartbeatRequestState, final BackgroundEventHandler backgroundEventHandler, + final String metricGroupPrefix, final Metrics metrics) { this.logger = logContext.logger(this.getClass()); this.maxPollIntervalMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); @@ -158,7 +160,7 @@ public HeartbeatRequestManager( this.membershipManager = membershipManager; this.backgroundEventHandler = backgroundEventHandler; this.pollTimer = timer; - this.heartbeatMetrics = createHeartbeatMetrics(metrics); + this.heartbeatMetrics = createHeartbeatMetrics(metrics, metricGroupPrefix); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index 7b3e75be0182c..b98f82d738e82 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -35,6 +35,7 @@ import java.util.function.Supplier; import static java.util.Objects.requireNonNull; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.common.utils.Utils.closeQuietly; /** @@ -191,6 +192,7 @@ protected RequestManagers create() { subscriptions, membershipManager, backgroundEventHandler, + CONSUMER_METRIC_GROUP_PREFIX, metrics); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index 62886c7996631..b747945b73a71 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -48,6 +48,7 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_INSTANCE_ID_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; @@ -226,6 +227,7 @@ public ConsumerTestBuilder(Optional groupInfo, boolean enableA heartbeatState, heartbeatRequestState, backgroundEventHandler, + CONSUMER_METRIC_GROUP_PREFIX, metrics)); this.coordinatorRequestManager = Optional.of(coordinator); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index 8857c0f6c7b44..41a31a7eb18f7 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -85,6 +85,7 @@ public class HeartbeatRequestManagerTest { private int maxPollIntervalMs = DEFAULT_MAX_POLL_INTERVAL_MS; private long retryBackoffMaxMs = DEFAULT_RETRY_BACKOFF_MAX_MS; private static final String DEFAULT_GROUP_ID = "groupId"; + private String metricGroupPrefix = CONSUMER_METRIC_GROUP_PREFIX; private ConsumerTestBuilder testBuilder; private Time time; @@ -737,6 +738,7 @@ private HeartbeatRequestManager createHeartbeatRequestManager( heartbeatState, heartbeatRequestState, backgroundEventHandler, + metricGroupPrefix, metrics); } } From d0e1728812cbd1c2ba0c435ffa4f3841718fe6eb Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Fri, 26 Jan 2024 14:31:20 -0800 Subject: [PATCH 17/28] clean up --- .../internals/HeartbeatRequestManager.java | 2 +- .../metrics/AbstractConsumerMetrics.java | 60 ++++++++++--------- .../internals/metrics/HeartbeatMetrics.java | 16 +++-- .../metrics/RebalanceCallbackMetrics.java | 14 ++--- .../internals/ConsumerMetricsTest.java | 41 ++++--------- .../HeartbeatRequestManagerTest.java | 2 +- 6 files changed, 65 insertions(+), 70 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 9ab81ccef3230..267b2c465d1b8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -428,7 +428,7 @@ private void handleFatalFailure(Throwable error) { membershipManager.transitionToFatal(); } - private HeartbeatMetrics createHeartbeatMetrics(Metrics metrics) { + private HeartbeatMetrics createHeartbeatMetrics(Metrics metrics, String metricGroupPrefix) { HeartbeatMetrics hbm = new HeartbeatMetrics(metrics); Measurable lastHeartbeat = (config, now) -> { final long lastHeartbeatSend = heartbeatRequestState.lastSentMs; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java index 56f1a310b3f0d..950fdf566cb0d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java @@ -22,74 +22,80 @@ import org.apache.kafka.common.metrics.stats.Meter; import org.apache.kafka.common.metrics.stats.WindowedCount; -import java.util.Objects; -import java.util.Optional; - import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.FETCH_MANAGER_METRICS_SUFFIX; /** - * This class encapsulates the creation of metrics for the kafka consumer. When constructing a metrics object, you - * need to provide the suffix of the metric group. These are: - *
    • - *
        Group Coordinator: {@link MetricSuffix#COORDINATOR}
      - *
        Consumer metrics: {@link MetricSuffix#CONSUMER}
      - *
        Fetch manager metrics: {@link MetricSuffix#FETCH_MANAGER}
      - *
      • + * Base class for different consumer metrics to extend. This class helps to construct the logical group name from the + * given prefix and suffix, and provides a few common utilities. + * + *

      + * Prefix can be provided by the user and is "consumer-" by default (see {@link ConsumerUtils#CONSUMER_METRIC_GROUP_PREFIX}). + *

      * - * If no prefix is provided, we will use the default prefix {@link ConsumerUtils#CONSUMER_METRIC_GROUP_PREFIX}. + *

      + * The suffix can be one of the following: + *

        + *
      • -coordinator-metrics: {@link MetricGroupSuffix#COORDINATOR}
        • + *
      • -metrics: {@link MetricGroupSuffix#CONSUMER}
        • + *
      • -fetch-manager-metrics: {@link MetricGroupSuffix#FETCH_MANAGER}
        • + *
      + *

      */ public abstract class AbstractConsumerMetrics { - protected final String groupMetricsName; + protected final String metricGroupName; - public enum MetricSuffix { + public enum MetricGroupSuffix { COORDINATOR(COORDINATOR_METRICS_SUFFIX), CONSUMER(CONSUMER_METRICS_SUFFIX), FETCH_MANAGER(FETCH_MANAGER_METRICS_SUFFIX); private final String suffix; - MetricSuffix(String suffix) { + MetricGroupSuffix(String suffix) { this.suffix = suffix; } - public String getSuffix() { + @Override + public String toString() { return suffix; } } - public AbstractConsumerMetrics(MetricSuffix suffix) { - this(Optional.empty(), suffix.getSuffix()); + public AbstractConsumerMetrics(MetricGroupSuffix suffix) { + this(CONSUMER_METRIC_GROUP_PREFIX, suffix.toString()); } - public AbstractConsumerMetrics(Optional grpMetricsPrefix, String suffix) { - Objects.requireNonNull(suffix, "suffix cannot be null"); - String prefix = grpMetricsPrefix.orElse(CONSUMER_METRIC_GROUP_PREFIX); - groupMetricsName = prefix + suffix; + public AbstractConsumerMetrics(String prefix, String suffix) { + if (suffix == null) + throw new IllegalArgumentException("metric group suffix cannot be null"); + if (prefix == null) + throw new IllegalArgumentException("metric group prefix cannot be null"); + metricGroupName = prefix + suffix; } - public String groupMetricsName() { - return groupMetricsName; + public String metricGroupName() { + return metricGroupName; } public Meter createMeter(Metrics metrics, String baseName, String descriptiveName) { return new Meter(new WindowedCount(), metrics.metricName(baseName + "-rate", - groupMetricsName, + metricGroupName, String.format("The number of %s per second", descriptiveName)), metrics.metricName(baseName + "-total", - groupMetricsName, + metricGroupName, String.format("The total number of %s", descriptiveName))); } public void addMetric(Metrics metrics, - String name, + String metricName, String description, Measurable measurable) { metrics.addMetric( - metrics.metricName(name, groupMetricsName, description), + metrics.metricName(metricName, metricGroupName, description), measurable); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java index ed9c93947ae5e..4ec2899188cf2 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java @@ -20,19 +20,25 @@ import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Max; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; + public class HeartbeatMetrics extends AbstractConsumerMetrics { public final Sensor heartbeatSensor; public HeartbeatMetrics(Metrics metrics) { - super(MetricSuffix.COORDINATOR); + this(metrics, CONSUMER_METRIC_GROUP_PREFIX); + } + + public HeartbeatMetrics(Metrics metrics, String prefix) { + super(MetricGroupSuffix.COORDINATOR); heartbeatSensor = metrics.sensor("heartbeat-latency"); heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", - groupMetricsName, + metricGroupName, "The max time taken to receive a response to a heartbeat request"), - new Max()); + new Max()); heartbeatSensor.add(createMeter(metrics, - "heartbeat", - "heartbeats")); + "heartbeat", + "heartbeats")); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java index c5850cd241a6e..b0c1bad324663 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java @@ -31,29 +31,29 @@ public RebalanceCallbackMetrics(Metrics metrics) { } public RebalanceCallbackMetrics(Metrics metrics, String grpMetricsPrefix) { - super(MetricSuffix.COORDINATOR); + super(MetricGroupSuffix.COORDINATOR); revokeCallbackSensor = metrics.sensor("partition-revoked-latency"); revokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-avg", - groupMetricsName, + metricGroupName, "The average time taken for a partition-revoked rebalance listener callback"), new Avg()); revokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-max", - groupMetricsName, + metricGroupName, "The max time taken for a partition-revoked rebalance listener callback"), new Max()); assignCallbackSensor = metrics.sensor("partition-assigned-latency"); assignCallbackSensor.add(metrics.metricName("partition-assigned-latency-avg", - groupMetricsName, + metricGroupName, "The average time taken for a partition-assigned rebalance listener callback"), new Avg()); assignCallbackSensor.add(metrics.metricName("partition-assigned-latency-max", - groupMetricsName, + metricGroupName, "The max time taken for a partition-assigned rebalance listener callback"), new Max()); loseCallbackSensor = metrics.sensor("partition-lost-latency"); loseCallbackSensor.add(metrics.metricName("partition-lost-latency-avg", - groupMetricsName, + metricGroupName, "The average time taken for a partition-lost rebalance listener callback"), new Avg()); loseCallbackSensor.add(metrics.metricName("partition-lost-latency-max", - groupMetricsName, + metricGroupName, "The max time taken for a partition-lost rebalance listener callback"), new Max()); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java index 9072575bada0e..343e74ce40965 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java @@ -25,14 +25,12 @@ import org.apache.kafka.common.utils.Time; import org.junit.jupiter.api.Test; -import java.util.Optional; - import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.FETCH_MANAGER_METRICS_SUFFIX; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; public class ConsumerMetricsTest { private Time time = new MockTime(); @@ -83,43 +81,28 @@ public void testHeartbeatMetrics() { @Test public void testMetricsGroupName() { // consumer-coordinator-metrics - NoopCoordinatorMetrics defaultCoordinatorMetrics = new NoopCoordinatorMetrics(Optional.empty()); - assertEquals(CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX, defaultCoordinatorMetrics.groupMetricsName()); - NoopCoordinatorMetrics coordinatorMetrics = new NoopCoordinatorMetrics(Optional.of("customCoordinatorPrefix")); - assertEquals("customCoordinatorPrefix" + COORDINATOR_METRICS_SUFFIX, coordinatorMetrics.groupMetricsName()); + NoopConsumerMetrics coordinatorMetrics = new NoopConsumerMetrics("customCoordinatorPrefix", AbstractConsumerMetrics.MetricGroupSuffix.COORDINATOR); + assertEquals("customCoordinatorPrefix" + COORDINATOR_METRICS_SUFFIX, coordinatorMetrics.metricGroupName()); // consumer-metrics - NoopConsumerMetrics defaultConsumerMetrics = new NoopConsumerMetrics(Optional.empty()); - assertEquals(CONSUMER_METRIC_GROUP_PREFIX + CONSUMER_METRICS_SUFFIX, defaultConsumerMetrics.groupMetricsName()); - NoopConsumerMetrics consumerMetrics = new NoopConsumerMetrics(Optional.of("customConsumerPrefix")); - assertEquals("customConsumerPrefix" + CONSUMER_METRICS_SUFFIX, consumerMetrics.groupMetricsName()); + NoopConsumerMetrics consumerMetrics = new NoopConsumerMetrics("customConsumerPrefix", AbstractConsumerMetrics.MetricGroupSuffix.CONSUMER); + assertEquals("customConsumerPrefix" + CONSUMER_METRICS_SUFFIX, consumerMetrics.metricGroupName()); // consumer-fetch-manager-metrics - NoopFetchManagerMetrics defaultFetchMetrics = new NoopFetchManagerMetrics(Optional.empty()); - assertEquals(CONSUMER_METRIC_GROUP_PREFIX + FETCH_MANAGER_METRICS_SUFFIX, defaultFetchMetrics.groupMetricsName()); - NoopFetchManagerMetrics fetchMetrics = new NoopFetchManagerMetrics(Optional.of("customFetchPrefix")); - assertEquals("customFetchPrefix" + FETCH_MANAGER_METRICS_SUFFIX, fetchMetrics.groupMetricsName()); + NoopConsumerMetrics fetchMetrics = new NoopConsumerMetrics("customFetchPrefix", AbstractConsumerMetrics.MetricGroupSuffix.FETCH_MANAGER); + assertEquals("customFetchPrefix" + FETCH_MANAGER_METRICS_SUFFIX, fetchMetrics.metricGroupName()); + + assertThrows(IllegalArgumentException.class, () -> new NoopConsumerMetrics(null, + AbstractConsumerMetrics.MetricGroupSuffix.CONSUMER)); } private KafkaMetric getMetric(String metricName, String groupName) { return metrics.metrics().get(metrics.metricName(metricName, groupName)); } - static class NoopCoordinatorMetrics extends AbstractConsumerMetrics { - public NoopCoordinatorMetrics(Optional prefix) { - super(prefix, MetricSuffix.COORDINATOR.getSuffix()); - } - } - static class NoopConsumerMetrics extends AbstractConsumerMetrics { - public NoopConsumerMetrics(Optional prefix) { - super(prefix, MetricSuffix.CONSUMER.getSuffix()); - } - } - - static class NoopFetchManagerMetrics extends AbstractConsumerMetrics { - public NoopFetchManagerMetrics(Optional prefix) { - super(prefix, MetricSuffix.FETCH_MANAGER.getSuffix()); + public NoopConsumerMetrics(String prefix, MetricGroupSuffix suffix) { + super(prefix, suffix.toString()); } } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index 41a31a7eb18f7..68822970c17eb 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -718,7 +718,7 @@ private ConsumerConfig config() { } private KafkaMetric getMetric(final String name) { - return metrics.metrics().get(metrics.metricName(name, CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX)); + return metrics.metrics().get(metrics.metricName(name, metricGroupPrefix + COORDINATOR_METRICS_SUFFIX)); } private HeartbeatRequestManager createHeartbeatRequestManager( From 2e425a6d88f4e2c9c419742c24b4410d200e8270 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Fri, 26 Jan 2024 15:39:38 -0800 Subject: [PATCH 18/28] Update AbstractConsumerMetrics.java --- .../consumer/internals/metrics/AbstractConsumerMetrics.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java index 950fdf566cb0d..3e27167ddc286 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; -import org.apache.kafka.clients.consumer.internals.ConsumerUtils; import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.stats.Meter; @@ -32,10 +31,6 @@ * given prefix and suffix, and provides a few common utilities. * *

      - * Prefix can be provided by the user and is "consumer-" by default (see {@link ConsumerUtils#CONSUMER_METRIC_GROUP_PREFIX}). - *

      - * - *

      * The suffix can be one of the following: *

        *
      • -coordinator-metrics: {@link MetricGroupSuffix#COORDINATOR}
        • From 308daa21c1a232870a486c1f1ba771b24367d0a5 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Sun, 28 Jan 2024 10:08:20 -0800 Subject: [PATCH 19/28] Reconsolidate metrics into consumer coordinator metrics clean up --- .../kafka/clients/consumer/KafkaConsumer.java | 2 +- .../internals/AsyncKafkaConsumer.java | 7 +- .../internals/CommitRequestManager.java | 35 ++------- .../internals/ConsumerCoordinator.java | 4 +- .../consumer/internals/ConsumerDelegate.java | 1 + .../ConsumerRebalanceListenerInvoker.java | 12 +-- .../consumer/internals/ConsumerUtils.java | 1 - .../internals/HeartbeatRequestManager.java | 35 ++------- .../internals/LegacyKafkaConsumer.java | 1 + .../consumer/internals/RequestManagers.java | 2 - ...va => AbstractConsumerMetricsManager.java} | 19 ++--- .../internals/metrics/HeartbeatMetrics.java | 44 ----------- .../metrics/HeartbeatMetricsManager.java | 69 +++++++++++++++++ .../{ => metrics}/KafkaConsumerMetrics.java | 10 +-- .../metrics/OffsetCommitMetricsManager.java | 50 +++++++++++++ .../metrics/RebalanceCallbackMetrics.java | 59 --------------- .../RebalanceCallbackMetricsManager.java | 74 +++++++++++++++++++ .../internals/CommitRequestManagerTest.java | 1 - ...t.java => ConsumerMetricsManagerTest.java} | 44 +++++------ .../internals/ConsumerTestBuilder.java | 6 +- .../HeartbeatRequestManagerTest.java | 6 +- .../internals/KafkaConsumerMetricsTest.java | 1 + .../internals/MembershipManagerImplTest.java | 4 +- 23 files changed, 259 insertions(+), 228 deletions(-) rename clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/{AbstractConsumerMetrics.java => AbstractConsumerMetricsManager.java} (81%) delete mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java rename clients/src/main/java/org/apache/kafka/clients/consumer/internals/{ => metrics}/KafkaConsumerMetrics.java (93%) create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java delete mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java rename clients/src/test/java/org/apache/kafka/clients/consumer/internals/{ConsumerMetricsTest.java => ConsumerMetricsManagerTest.java} (75%) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index 82e1f8b93da3c..bb75a40657311 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -20,7 +20,7 @@ import org.apache.kafka.clients.consumer.internals.ConsumerDelegate; import org.apache.kafka.clients.consumer.internals.ConsumerDelegateCreator; import org.apache.kafka.clients.consumer.internals.ConsumerMetadata; -import org.apache.kafka.clients.consumer.internals.KafkaConsumerMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.KafkaConsumerMetrics; import org.apache.kafka.clients.consumer.internals.SubscriptionState; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java index c6b9efa23aef5..9198039a85a44 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java @@ -60,7 +60,8 @@ import org.apache.kafka.clients.consumer.internals.events.TopicMetadataApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.UnsubscribeApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent; -import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.KafkaConsumerMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.KafkaException; @@ -395,7 +396,7 @@ private void process(final ConsumerRebalanceListenerCallbackNeededEvent event) { logContext, subscriptions, time, - new RebalanceCallbackMetrics(metrics) + new RebalanceCallbackMetricsManager(metrics) ); this.backgroundEventProcessor = new BackgroundEventProcessor( logContext, @@ -538,7 +539,7 @@ private void process(final ConsumerRebalanceListenerCallbackNeededEvent event) { logContext, subscriptions, time, - new RebalanceCallbackMetrics(metrics) + new RebalanceCallbackMetricsManager(metrics, CONSUMER_METRIC_GROUP_PREFIX) ); ApiVersions apiVersions = new ApiVersions(); Supplier networkClientDelegateSupplier = () -> new NetworkClientDelegate( diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java index 08cc6638aad8c..72fd1dbfed0ae 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.RetriableCommitFailedException; +import org.apache.kafka.clients.consumer.internals.metrics.OffsetCommitMetricsManager; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.DisconnectException; @@ -32,11 +33,6 @@ import org.apache.kafka.common.message.OffsetCommitRequestData; import org.apache.kafka.common.message.OffsetCommitResponseData; import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Avg; -import org.apache.kafka.common.metrics.stats.Max; -import org.apache.kafka.common.metrics.stats.Meter; -import org.apache.kafka.common.metrics.stats.WindowedCount; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.requests.AbstractRequest; @@ -66,8 +62,6 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED; import static org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult.EMPTY; import static org.apache.kafka.common.protocol.Errors.COORDINATOR_LOAD_IN_PROGRESS; @@ -87,7 +81,7 @@ public class CommitRequestManager implements RequestManager, MemberStateListener private final boolean throwOnFetchStableOffsetUnsupported; final PendingRequests pendingRequests; private boolean closing = false; - private Sensor commitSensor; + private OffsetCommitMetricsManager offsetCommitMetricsManager; /** * Latest member ID and epoch received via the {@link #onMemberEpochUpdated(Optional, Optional)}, @@ -116,7 +110,6 @@ public CommitRequestManager( config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG), config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG), OptionalDouble.empty(), - CONSUMER_METRIC_GROUP_PREFIX, metrics); } @@ -132,7 +125,6 @@ public CommitRequestManager( final long retryBackoffMs, final long retryBackoffMaxMs, final OptionalDouble jitter, - final String metricGroupPrefix, final Metrics metrics) { Objects.requireNonNull(coordinatorRequestManager, "Coordinator is needed upon committing offsets"); this.logContext = logContext; @@ -154,7 +146,7 @@ public CommitRequestManager( this.jitter = jitter; this.throwOnFetchStableOffsetUnsupported = config.getBoolean(THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED); this.memberInfo = new MemberInfo(); - this.commitSensor = addCommitSensor(metrics, metricGroupPrefix); + this.offsetCommitMetricsManager = new OffsetCommitMetricsManager(metrics); } /** @@ -394,23 +386,6 @@ public NetworkClientDelegate.PollResult drainPendingOffsetCommitRequests() { return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, requests); } - private Sensor addCommitSensor(Metrics metrics, String metricGrpPrefix) { - String metricGrpName = metricGrpPrefix + COORDINATOR_METRICS_SUFFIX; - Sensor sensor = metrics.sensor("commit-latency"); - sensor.add(metrics.metricName("commit-latency-avg", - metricGrpName, - "The average time taken for a commit request"), new Avg()); - sensor.add(metrics.metricName("commit-latency-max", - metricGrpName, - "The max time taken for a commit request"), new Max()); - sensor.add(new Meter(new WindowedCount(), - metrics.metricName("commit-rate", metricGrpName, - "The number of commit calls per second"), - metrics.metricName("commit-total", metricGrpName, - "The total number of commit calls"))); - return sensor; - } - private class OffsetCommitRequestState extends RetriableRequestState { private final Map offsets; private final String groupId; @@ -505,8 +480,8 @@ public NetworkClientDelegate.UnsentRequest toUnsentRequest() { * - fail the future with a non-recoverable KafkaException for all unexpected errors (even if retriable) */ @Override - void onResponse(final ClientResponse response) { - commitSensor.record(response.requestLatencyMs()); + public void onResponse(final ClientResponse response) { + offsetCommitMetricsManager.recordRequestLatency(response.requestLatencyMs()); long currentTimeMs = response.receivedTimeMs(); OffsetCommitResponse commitResponse = (OffsetCommitResponse) response.responseBody(); Set unauthorizedTopics = new HashSet<>(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 398baf7fb079c..5af8633806da8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -29,7 +29,7 @@ import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.clients.consumer.RetriableCommitFailedException; import org.apache.kafka.clients.consumer.internals.Utils.TopicPartitionComparator; -import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Node; @@ -235,7 +235,7 @@ public ConsumerCoordinator(GroupRebalanceConfig rebalanceConfig, logContext, subscriptions, time, - new RebalanceCallbackMetrics(metrics, metricGrpPrefix) + new RebalanceCallbackMetricsManager(metrics, metricGrpPrefix) ); this.metadata.requestUpdate(true); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegate.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegate.java index 612827ebe83df..67e8d991d639f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegate.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegate.java @@ -17,6 +17,7 @@ package org.apache.kafka.clients.consumer.internals; import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.internals.metrics.KafkaConsumerMetrics; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.utils.Timer; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java index ea60436deb1ed..ca68158d4cf3b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java @@ -17,7 +17,7 @@ package org.apache.kafka.clients.consumer.internals; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; -import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.errors.WakeupException; @@ -41,12 +41,12 @@ public class ConsumerRebalanceListenerInvoker { private final Logger log; private final SubscriptionState subscriptions; private final Time time; - private final RebalanceCallbackMetrics metrics; + private final RebalanceCallbackMetricsManager metrics; ConsumerRebalanceListenerInvoker(LogContext logContext, SubscriptionState subscriptions, Time time, - RebalanceCallbackMetrics metrics) { + RebalanceCallbackMetricsManager metrics) { this.log = logContext.logger(getClass()); this.subscriptions = subscriptions; this.time = time; @@ -62,7 +62,7 @@ public Exception invokePartitionsAssigned(final SortedSet assign try { final long startMs = time.milliseconds(); listener.get().onPartitionsAssigned(assignedPartitions); - metrics.assignCallbackSensor.record(time.milliseconds() - startMs); + metrics.recordPartitionAssignLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { @@ -88,7 +88,7 @@ public Exception invokePartitionsRevoked(final SortedSet revoked try { final long startMs = time.milliseconds(); listener.get().onPartitionsRevoked(revokedPartitions); - metrics.revokeCallbackSensor.record(time.milliseconds() - startMs); + metrics.recordPartitionRevokeLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { @@ -114,7 +114,7 @@ public Exception invokePartitionsLost(final SortedSet lostPartit try { final long startMs = time.milliseconds(); listener.get().onPartitionsLost(lostPartitions); - metrics.loseCallbackSensor.record(time.milliseconds() - startMs); + metrics.recordPartitionLostLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java index ddb79b034c872..5a7745b40dd0d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java @@ -67,7 +67,6 @@ public final class ConsumerUtils { public static final String CONSUMER_METRIC_GROUP_PREFIX = "consumer"; public static final String COORDINATOR_METRICS_SUFFIX = "-coordinator-metrics"; public static final String CONSUMER_METRICS_SUFFIX = "-metrics"; - public static final String FETCH_MANAGER_METRICS_SUFFIX = "-fetch-manager-metrics"; /** * A fixed, large enough value will suffice for max. diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 267b2c465d1b8..2e0e63f786e40 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -23,12 +23,11 @@ import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.GroupMetadataUpdateEvent; -import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetricsManager; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.GroupAuthorizationException; import org.apache.kafka.common.errors.RetriableException; import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; -import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; @@ -44,7 +43,6 @@ import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -112,7 +110,7 @@ public class HeartbeatRequestManager implements RequestManager { * sending heartbeat until the next poll. */ private final Timer pollTimer; - private final HeartbeatMetrics heartbeatMetrics; + private final HeartbeatMetricsManager heartbeatMetricsManager; private GroupMetadataUpdateEvent previousGroupMetadataUpdateEvent = null; @@ -124,7 +122,6 @@ public HeartbeatRequestManager( final SubscriptionState subscriptions, final MembershipManager membershipManager, final BackgroundEventHandler backgroundEventHandler, - final String metricGroupPrefix, final Metrics metrics) { this.coordinatorRequestManager = coordinatorRequestManager; this.logger = logContext.logger(getClass()); @@ -137,7 +134,7 @@ public HeartbeatRequestManager( this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, retryBackoffMaxMs, maxPollIntervalMs); this.pollTimer = time.timer(maxPollIntervalMs); - this.heartbeatMetrics = createHeartbeatMetrics(metrics, metricGroupPrefix); + this.heartbeatMetricsManager = new HeartbeatMetricsManager(metrics); } // Visible for testing @@ -150,7 +147,6 @@ public HeartbeatRequestManager( final HeartbeatState heartbeatState, final HeartbeatRequestState heartbeatRequestState, final BackgroundEventHandler backgroundEventHandler, - final String metricGroupPrefix, final Metrics metrics) { this.logger = logContext.logger(this.getClass()); this.maxPollIntervalMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); @@ -160,7 +156,7 @@ public HeartbeatRequestManager( this.membershipManager = membershipManager; this.backgroundEventHandler = backgroundEventHandler; this.pollTimer = timer; - this.heartbeatMetrics = createHeartbeatMetrics(metrics, metricGroupPrefix); + this.heartbeatMetricsManager = new HeartbeatMetricsManager(metrics); } /** @@ -256,6 +252,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final long curr NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(ignoreResponse); heartbeatRequestState.onSendAttempt(currentTimeMs); membershipManager.onHeartbeatRequestSent(); + heartbeatMetricsManager.recordHeartbeatSentMs(currentTimeMs); return request; } @@ -268,7 +265,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i else return request.whenComplete((response, exception) -> { if (response != null) { - heartbeatMetrics.heartbeatSensor.record(response.requestLatencyMs()); + heartbeatMetricsManager.recordRequestLatency(response.requestLatencyMs()); onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), request.handler().completionTimeMs()); } else { onFailure(exception, request.handler().completionTimeMs()); @@ -279,7 +276,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i private NetworkClientDelegate.UnsentRequest logResponse(final NetworkClientDelegate.UnsentRequest request) { return request.whenComplete((response, exception) -> { if (response != null) { - heartbeatMetrics.heartbeatSensor.record(response.requestLatencyMs()); + heartbeatMetricsManager.recordRequestLatency(response.requestLatencyMs()); Errors error = Errors.forCode(((ConsumerGroupHeartbeatResponse) response.responseBody()).data().errorCode()); if (error == Errors.NONE) @@ -428,24 +425,6 @@ private void handleFatalFailure(Throwable error) { membershipManager.transitionToFatal(); } - private HeartbeatMetrics createHeartbeatMetrics(Metrics metrics, String metricGroupPrefix) { - HeartbeatMetrics hbm = new HeartbeatMetrics(metrics); - Measurable lastHeartbeat = (config, now) -> { - final long lastHeartbeatSend = heartbeatRequestState.lastSentMs; - if (lastHeartbeatSend < 0L) - // if no heartbeat is ever triggered, just return -1. - return -1d; - else - return TimeUnit.SECONDS.convert(now - lastHeartbeatSend, TimeUnit.MILLISECONDS); - }; - hbm.addMetric( - metrics, - "last-heartbeat-seconds-ago", - "The number of seconds since the last coordinator heartbeat was sent", - lastHeartbeat); - return hbm; - } - /** * Represents the state of a heartbeat request, including logic for timing, retries, and exponential backoff. The * object extends {@link RequestState} to enable exponential backoff and duplicated request handling. The two fields diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/LegacyKafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/LegacyKafkaConsumer.java index bcc8cb40f2d91..80be0959d69cb 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/LegacyKafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/LegacyKafkaConsumer.java @@ -35,6 +35,7 @@ import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.clients.consumer.internals.metrics.KafkaConsumerMetrics; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.KafkaException; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index b98f82d738e82..7b3e75be0182c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -35,7 +35,6 @@ import java.util.function.Supplier; import static java.util.Objects.requireNonNull; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.common.utils.Utils.closeQuietly; /** @@ -192,7 +191,6 @@ protected RequestManagers create() { subscriptions, membershipManager, backgroundEventHandler, - CONSUMER_METRIC_GROUP_PREFIX, metrics); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java similarity index 81% rename from clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java rename to clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java index 3e27167ddc286..1eafcfa95bfbc 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java @@ -22,9 +22,7 @@ import org.apache.kafka.common.metrics.stats.WindowedCount; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.FETCH_MANAGER_METRICS_SUFFIX; /** * Base class for different consumer metrics to extend. This class helps to construct the logical group name from the @@ -35,17 +33,16 @@ *
          *
        • -coordinator-metrics: {@link MetricGroupSuffix#COORDINATOR}
          • *
        • -metrics: {@link MetricGroupSuffix#CONSUMER}
          • - *
        • -fetch-manager-metrics: {@link MetricGroupSuffix#FETCH_MANAGER}
          • *
        *

        */ -public abstract class AbstractConsumerMetrics { +public abstract class AbstractConsumerMetricsManager { protected final String metricGroupName; + final Metrics metrics; public enum MetricGroupSuffix { COORDINATOR(COORDINATOR_METRICS_SUFFIX), - CONSUMER(CONSUMER_METRICS_SUFFIX), - FETCH_MANAGER(FETCH_MANAGER_METRICS_SUFFIX); + CONSUMER(CONSUMER_METRICS_SUFFIX); private final String suffix; @@ -59,15 +56,12 @@ public String toString() { } } - public AbstractConsumerMetrics(MetricGroupSuffix suffix) { - this(CONSUMER_METRIC_GROUP_PREFIX, suffix.toString()); - } - - public AbstractConsumerMetrics(String prefix, String suffix) { + public AbstractConsumerMetricsManager(Metrics metrics, String prefix, MetricGroupSuffix suffix) { if (suffix == null) throw new IllegalArgumentException("metric group suffix cannot be null"); if (prefix == null) throw new IllegalArgumentException("metric group prefix cannot be null"); + this.metrics = metrics; metricGroupName = prefix + suffix; } @@ -85,8 +79,7 @@ public Meter createMeter(Metrics metrics, String baseName, String descriptiveNam String.format("The total number of %s", descriptiveName))); } - public void addMetric(Metrics metrics, - String metricName, + public void addMetric(String metricName, String description, Measurable measurable) { metrics.addMetric( diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java deleted file mode 100644 index 4ec2899188cf2..0000000000000 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetrics.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals.metrics; - -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Max; - -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; - -public class HeartbeatMetrics extends AbstractConsumerMetrics { - - public final Sensor heartbeatSensor; - - public HeartbeatMetrics(Metrics metrics) { - this(metrics, CONSUMER_METRIC_GROUP_PREFIX); - } - - public HeartbeatMetrics(Metrics metrics, String prefix) { - super(MetricGroupSuffix.COORDINATOR); - heartbeatSensor = metrics.sensor("heartbeat-latency"); - heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", - metricGroupName, - "The max time taken to receive a response to a heartbeat request"), - new Max()); - heartbeatSensor.add(createMeter(metrics, - "heartbeat", - "heartbeats")); - } -} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java new file mode 100644 index 0000000000000..708d89ea2d510 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.metrics; + +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Max; + +import java.util.concurrent.TimeUnit; + +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; + +public class HeartbeatMetricsManager extends AbstractConsumerMetricsManager { + + public final Sensor heartbeatSensor; + private long lastHeartbeatMs = -1L; + + public HeartbeatMetricsManager(Metrics metrics) { + this(metrics, CONSUMER_METRIC_GROUP_PREFIX); + } + + public HeartbeatMetricsManager(Metrics metrics, String prefix) { + super(metrics, prefix, MetricGroupSuffix.COORDINATOR); + heartbeatSensor = metrics.sensor("heartbeat-latency"); + heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", + metricGroupName, + "The max time taken to receive a response to a heartbeat request"), + new Max()); + heartbeatSensor.add(createMeter(metrics, + "heartbeat", + "heartbeats")); + + Measurable lastHeartbeat = (config, now) -> { + final long lastHeartbeatSend = lastHeartbeatMs; + if (lastHeartbeatSend < 0L) + // if no heartbeat is ever triggered, just return -1. + return -1d; + else + return TimeUnit.SECONDS.convert(now - lastHeartbeatSend, TimeUnit.MILLISECONDS); + }; + addMetric( + "last-heartbeat-seconds-ago", + "The number of seconds since the last coordinator heartbeat was sent", + lastHeartbeat); + } + + public void recordHeartbeatSentMs(long lastHeartbeatMs) { + this.lastHeartbeatMs = lastHeartbeatMs; + } + + public void recordRequestLatency(long requestLatencyMs) { + heartbeatSensor.record(requestLatencyMs); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java similarity index 93% rename from clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java rename to clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java index 0dc8a33b11bbb..ce32b9d97e02b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.clients.consumer.internals; +package org.apache.kafka.clients.consumer.internals.metrics; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.metrics.Measurable; @@ -26,20 +26,20 @@ import java.util.concurrent.TimeUnit; -public class KafkaConsumerMetrics implements AutoCloseable { +import static org.apache.kafka.clients.consumer.internals.metrics.AbstractConsumerMetricsManager.MetricGroupSuffix.CONSUMER; + +public class KafkaConsumerMetrics extends AbstractConsumerMetricsManager implements AutoCloseable { private final MetricName lastPollMetricName; private final Sensor timeBetweenPollSensor; private final Sensor pollIdleSensor; private final Sensor committedSensor; private final Sensor commitSyncSensor; - private final Metrics metrics; private long lastPollMs; private long pollStartMs; private long timeSinceLastPollMs; public KafkaConsumerMetrics(Metrics metrics, String metricGrpPrefix) { - this.metrics = metrics; - String metricGroupName = metricGrpPrefix + "-metrics"; + super(metrics, metricGrpPrefix, CONSUMER); Measurable lastPoll = (mConfig, now) -> { if (lastPollMs == 0L) // if no poll is ever triggered, just return -1. diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java new file mode 100644 index 0000000000000..7aef981066820 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.metrics; + +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; +import org.apache.kafka.common.metrics.stats.Max; +import org.apache.kafka.common.metrics.stats.Meter; +import org.apache.kafka.common.metrics.stats.WindowedCount; + +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; + +public class OffsetCommitMetricsManager extends AbstractConsumerMetricsManager { + private final Sensor commitSensor; + + public OffsetCommitMetricsManager(Metrics metrics) { + super(metrics, CONSUMER_METRIC_GROUP_PREFIX, MetricGroupSuffix.COORDINATOR); + commitSensor = metrics.sensor("commit-latency"); + commitSensor.add(metrics.metricName("commit-latency-avg", + metricGroupName(), + "The average time taken for a commit request"), new Avg()); + commitSensor.add(metrics.metricName("commit-latency-max", + metricGroupName(), + "The max time taken for a commit request"), new Max()); + commitSensor.add(new Meter(new WindowedCount(), + metrics.metricName("commit-rate", metricGroupName(), + "The number of commit calls per second"), + metrics.metricName("commit-total", metricGroupName(), + "The total number of commit calls"))); + } + + public void recordRequestLatency(long latencyMs) { + this.commitSensor.record(latencyMs); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java deleted file mode 100644 index b0c1bad324663..0000000000000 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetrics.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals.metrics; - -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Avg; -import org.apache.kafka.common.metrics.stats.Max; - -public class RebalanceCallbackMetrics extends AbstractConsumerMetrics { - public final Sensor revokeCallbackSensor; - public final Sensor assignCallbackSensor; - public final Sensor loseCallbackSensor; - - public RebalanceCallbackMetrics(Metrics metrics) { - this(metrics, null); - } - - public RebalanceCallbackMetrics(Metrics metrics, String grpMetricsPrefix) { - super(MetricGroupSuffix.COORDINATOR); - revokeCallbackSensor = metrics.sensor("partition-revoked-latency"); - revokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-avg", - metricGroupName, - "The average time taken for a partition-revoked rebalance listener callback"), new Avg()); - revokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-max", - metricGroupName, - "The max time taken for a partition-revoked rebalance listener callback"), new Max()); - - assignCallbackSensor = metrics.sensor("partition-assigned-latency"); - assignCallbackSensor.add(metrics.metricName("partition-assigned-latency-avg", - metricGroupName, - "The average time taken for a partition-assigned rebalance listener callback"), new Avg()); - assignCallbackSensor.add(metrics.metricName("partition-assigned-latency-max", - metricGroupName, - "The max time taken for a partition-assigned rebalance listener callback"), new Max()); - - loseCallbackSensor = metrics.sensor("partition-lost-latency"); - loseCallbackSensor.add(metrics.metricName("partition-lost-latency-avg", - metricGroupName, - "The average time taken for a partition-lost rebalance listener callback"), new Avg()); - loseCallbackSensor.add(metrics.metricName("partition-lost-latency-max", - metricGroupName, - "The max time taken for a partition-lost rebalance listener callback"), new Max()); - } -} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java new file mode 100644 index 0000000000000..19e7267805a0a --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.metrics; + +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; +import org.apache.kafka.common.metrics.stats.Max; + +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.metrics.AbstractConsumerMetricsManager.MetricGroupSuffix.COORDINATOR; + +public class RebalanceCallbackMetricsManager extends AbstractConsumerMetricsManager { + private final Sensor partitionRevokeCallbackSensor; + private final Sensor partitionAssignCallbackSensor; + private final Sensor partitionLostCallbackSensor; + + public RebalanceCallbackMetricsManager(Metrics metrics) { + this(metrics, CONSUMER_METRIC_GROUP_PREFIX); + } + + public RebalanceCallbackMetricsManager(Metrics metrics, String grpMetricsPrefix) { + super(metrics, grpMetricsPrefix, COORDINATOR); + partitionRevokeCallbackSensor = metrics.sensor("partition-revoked-latency"); + partitionRevokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-avg", + metricGroupName, + "The average time taken for a partition-revoked rebalance listener callback"), new Avg()); + partitionRevokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-max", + metricGroupName, + "The max time taken for a partition-revoked rebalance listener callback"), new Max()); + + partitionAssignCallbackSensor = metrics.sensor("partition-assigned-latency"); + partitionAssignCallbackSensor.add(metrics.metricName("partition-assigned-latency-avg", + metricGroupName, + "The average time taken for a partition-assigned rebalance listener callback"), new Avg()); + partitionAssignCallbackSensor.add(metrics.metricName("partition-assigned-latency-max", + metricGroupName, + "The max time taken for a partition-assigned rebalance listener callback"), new Max()); + + partitionLostCallbackSensor = metrics.sensor("partition-lost-latency"); + partitionLostCallbackSensor.add(metrics.metricName("partition-lost-latency-avg", + metricGroupName, + "The average time taken for a partition-lost rebalance listener callback"), new Avg()); + partitionLostCallbackSensor.add(metrics.metricName("partition-lost-latency-max", + metricGroupName, + "The max time taken for a partition-lost rebalance listener callback"), new Max()); + } + + public void recordPartitionRevokeLatency(long latencyMs) { + partitionRevokeCallbackSensor.record(latencyMs); + } + + public void recordPartitionAssignLatency(long latencyMs) { + partitionAssignCallbackSensor.record(latencyMs); + } + + public void recordPartitionLostLatency(long latencyMs) { + partitionLostCallbackSensor.record(latencyMs); + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java index f7a5e6fd114cf..139d922efe2c6 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java @@ -1079,7 +1079,6 @@ private CommitRequestManager create(final boolean autoCommitEnabled, final long retryBackoffMs, retryBackoffMaxMs, OptionalDouble.of(0), - consumerMetricGroupPrefix, metrics)); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java similarity index 75% rename from clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java rename to clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java index 343e74ce40965..5fad34987eeef 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java @@ -16,9 +16,9 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.consumer.internals.metrics.AbstractConsumerMetrics; -import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetrics; -import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.AbstractConsumerMetricsManager; +import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetricsManager; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; import org.apache.kafka.common.metrics.KafkaMetric; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.utils.MockTime; @@ -27,19 +27,18 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.FETCH_MANAGER_METRICS_SUFFIX; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; -public class ConsumerMetricsTest { +public class ConsumerMetricsManagerTest { private Time time = new MockTime(); private Metrics metrics = new Metrics(time); @Test public void testRebalanceCallbackMetrics() { // MBean: kafka.consumer:type=consumer-coordinator-metrics,client-id={clientId} - RebalanceCallbackMetrics rebalanceCallbackMetrics = new RebalanceCallbackMetrics(metrics); + RebalanceCallbackMetricsManager rebalanceCallbackMetricsManager = new RebalanceCallbackMetricsManager(metrics); String groupName = "consumer-coordinator-metrics"; assertNotNull(getMetric("partition-revoked-latency-avg", groupName)); assertNotNull(getMetric("partition-revoked-latency-max", groupName)); @@ -48,9 +47,9 @@ public void testRebalanceCallbackMetrics() { assertNotNull(getMetric("partition-lost-latency-avg", groupName)); assertNotNull(getMetric("partition-lost-latency-max", groupName)); - rebalanceCallbackMetrics.assignCallbackSensor.record(100); - rebalanceCallbackMetrics.revokeCallbackSensor.record(101); - rebalanceCallbackMetrics.loseCallbackSensor.record(102); + rebalanceCallbackMetricsManager.recordPartitionAssignLatency(100); + rebalanceCallbackMetricsManager.recordPartitionRevokeLatency(101); + rebalanceCallbackMetricsManager.recordPartitionLostLatency(102); assertEquals(101d, getMetric("partition-revoked-latency-avg", groupName).metricValue()); assertEquals(101d, getMetric("partition-revoked-latency-max", groupName).metricValue()); @@ -63,15 +62,15 @@ public void testRebalanceCallbackMetrics() { @Test public void testHeartbeatMetrics() { // MBean: kafka.consumer:type=consumer-coordinator-metrics,client-id={clientId} - HeartbeatMetrics heartbeatMetrics = new HeartbeatMetrics(metrics); + HeartbeatMetricsManager heartbeatMetricsManager = new HeartbeatMetricsManager(metrics); String groupName = "consumer-coordinator-metrics"; assertNotNull(getMetric("heartbeat-response-time-max", groupName)); assertNotNull(getMetric("heartbeat-rate", groupName)); assertNotNull(getMetric("heartbeat-total", groupName)); - heartbeatMetrics.heartbeatSensor.record(100); - heartbeatMetrics.heartbeatSensor.record(100); - heartbeatMetrics.heartbeatSensor.record(100); + heartbeatMetricsManager.heartbeatSensor.record(100); + heartbeatMetricsManager.heartbeatSensor.record(100); + heartbeatMetricsManager.heartbeatSensor.record(100); assertEquals(100d, getMetric("heartbeat-response-time-max", groupName).metricValue()); assertEquals(0.1d, (double) getMetric("heartbeat-rate", groupName).metricValue(), 0.01d); @@ -81,28 +80,25 @@ public void testHeartbeatMetrics() { @Test public void testMetricsGroupName() { // consumer-coordinator-metrics - NoopConsumerMetrics coordinatorMetrics = new NoopConsumerMetrics("customCoordinatorPrefix", AbstractConsumerMetrics.MetricGroupSuffix.COORDINATOR); + NoopConsumerMetrics coordinatorMetrics = new NoopConsumerMetrics(metrics, "customCoordinatorPrefix", AbstractConsumerMetricsManager.MetricGroupSuffix.COORDINATOR); assertEquals("customCoordinatorPrefix" + COORDINATOR_METRICS_SUFFIX, coordinatorMetrics.metricGroupName()); // consumer-metrics - NoopConsumerMetrics consumerMetrics = new NoopConsumerMetrics("customConsumerPrefix", AbstractConsumerMetrics.MetricGroupSuffix.CONSUMER); + NoopConsumerMetrics consumerMetrics = new NoopConsumerMetrics(metrics, "customConsumerPrefix", AbstractConsumerMetricsManager.MetricGroupSuffix.CONSUMER); assertEquals("customConsumerPrefix" + CONSUMER_METRICS_SUFFIX, consumerMetrics.metricGroupName()); - // consumer-fetch-manager-metrics - NoopConsumerMetrics fetchMetrics = new NoopConsumerMetrics("customFetchPrefix", AbstractConsumerMetrics.MetricGroupSuffix.FETCH_MANAGER); - assertEquals("customFetchPrefix" + FETCH_MANAGER_METRICS_SUFFIX, fetchMetrics.metricGroupName()); - - assertThrows(IllegalArgumentException.class, () -> new NoopConsumerMetrics(null, - AbstractConsumerMetrics.MetricGroupSuffix.CONSUMER)); + assertThrows(IllegalArgumentException.class, () -> new NoopConsumerMetrics(metrics, + null, + AbstractConsumerMetricsManager.MetricGroupSuffix.CONSUMER)); } private KafkaMetric getMetric(String metricName, String groupName) { return metrics.metrics().get(metrics.metricName(metricName, groupName)); } - static class NoopConsumerMetrics extends AbstractConsumerMetrics { - public NoopConsumerMetrics(String prefix, MetricGroupSuffix suffix) { - super(prefix, suffix.toString()); + static class NoopConsumerMetrics extends AbstractConsumerMetricsManager { + public NoopConsumerMetrics(Metrics metrics, String prefix, MetricGroupSuffix suffix) { + super(metrics, prefix, suffix); } } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index b747945b73a71..06bcd22bb597f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -25,7 +25,7 @@ import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; -import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.requests.MetadataResponse; @@ -48,7 +48,6 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_INSTANCE_ID_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; @@ -227,7 +226,6 @@ public ConsumerTestBuilder(Optional groupInfo, boolean enableA heartbeatState, heartbeatRequestState, backgroundEventHandler, - CONSUMER_METRIC_GROUP_PREFIX, metrics)); this.coordinatorRequestManager = Optional.of(coordinator); @@ -274,7 +272,7 @@ public ConsumerTestBuilder(Optional groupInfo, boolean enableA logContext, subscriptions, time, - new RebalanceCallbackMetrics(metrics) + new RebalanceCallbackMetricsManager(metrics) ); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index 68822970c17eb..33d54ab66389d 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -57,6 +57,7 @@ import java.util.Optional; import java.util.Properties; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_GROUP_INSTANCE_ID; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_HEARTBEAT_INTERVAL_MS; @@ -599,13 +600,13 @@ public void testHeartbeatMetrics() { assertHeartbeat(heartbeatRequestManager, 0); time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(1.0, getMetric("heartbeat-total").metricValue()); - assertEquals(heartbeatIntervalMs / 1000d, getMetric("last-heartbeat-seconds-ago").metricValue()); + assertEquals((double) TimeUnit.MILLISECONDS.toSeconds(heartbeatIntervalMs), getMetric("last-heartbeat-seconds-ago").metricValue()); assertHeartbeat(heartbeatRequestManager, DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(0.06d, (double) getMetric("heartbeat-rate").metricValue(), 0.005d); assertEquals(2.0, getMetric("heartbeat-total").metricValue()); time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); - assertEquals(heartbeatIntervalMs / 1000d, getMetric("last-heartbeat-seconds-ago").metricValue()); + assertEquals((double) TimeUnit.MILLISECONDS.toSeconds(heartbeatIntervalMs), getMetric("last-heartbeat-seconds-ago").metricValue()); } private void assertHeartbeat(HeartbeatRequestManager hrm, int nextPollMs) { @@ -738,7 +739,6 @@ private HeartbeatRequestManager createHeartbeatRequestManager( heartbeatState, heartbeatRequestState, backgroundEventHandler, - metricGroupPrefix, metrics); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetricsTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetricsTest.java index 087f90b7efa31..c75ee906e535b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetricsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetricsTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.clients.consumer.internals.metrics.KafkaConsumerMetrics; import org.apache.kafka.common.metrics.Metrics; import org.junit.jupiter.api.Test; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java index 7683a521e0d6a..a87b6efe16152 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java @@ -20,7 +20,7 @@ import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.clients.consumer.internals.events.ConsumerRebalanceListenerCallbackCompletedEvent; import org.apache.kafka.clients.consumer.internals.events.ConsumerRebalanceListenerCallbackNeededEvent; -import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.TopicIdPartition; import org.apache.kafka.common.TopicPartition; @@ -1388,7 +1388,7 @@ private ConsumerRebalanceListenerInvoker consumerRebalanceListenerInvoker() { new LogContext(), subscriptionState, new MockTime(1), - new RebalanceCallbackMetrics(new Metrics()) + new RebalanceCallbackMetricsManager(new Metrics()) ); } From 882460887ce1969237bf5553ee0072ef1ddbdb7c Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Sun, 28 Jan 2024 17:04:37 -0800 Subject: [PATCH 20/28] clean up name --- .../consumer/internals/CommitRequestManager.java | 6 +++--- .../internals/ConsumerRebalanceListenerInvoker.java | 12 ++++++------ .../consumer/internals/HeartbeatRequestManager.java | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java index 72fd1dbfed0ae..58d1ffd2d3c6d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java @@ -81,7 +81,7 @@ public class CommitRequestManager implements RequestManager, MemberStateListener private final boolean throwOnFetchStableOffsetUnsupported; final PendingRequests pendingRequests; private boolean closing = false; - private OffsetCommitMetricsManager offsetCommitMetricsManager; + private OffsetCommitMetricsManager metricsManager; /** * Latest member ID and epoch received via the {@link #onMemberEpochUpdated(Optional, Optional)}, @@ -146,7 +146,7 @@ public CommitRequestManager( this.jitter = jitter; this.throwOnFetchStableOffsetUnsupported = config.getBoolean(THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED); this.memberInfo = new MemberInfo(); - this.offsetCommitMetricsManager = new OffsetCommitMetricsManager(metrics); + this.metricsManager = new OffsetCommitMetricsManager(metrics); } /** @@ -481,7 +481,7 @@ public NetworkClientDelegate.UnsentRequest toUnsentRequest() { */ @Override public void onResponse(final ClientResponse response) { - offsetCommitMetricsManager.recordRequestLatency(response.requestLatencyMs()); + metricsManager.recordRequestLatency(response.requestLatencyMs()); long currentTimeMs = response.receivedTimeMs(); OffsetCommitResponse commitResponse = (OffsetCommitResponse) response.responseBody(); Set unauthorizedTopics = new HashSet<>(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java index ca68158d4cf3b..93d5c65bee539 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java @@ -41,16 +41,16 @@ public class ConsumerRebalanceListenerInvoker { private final Logger log; private final SubscriptionState subscriptions; private final Time time; - private final RebalanceCallbackMetricsManager metrics; + private final RebalanceCallbackMetricsManager metricsManager; ConsumerRebalanceListenerInvoker(LogContext logContext, SubscriptionState subscriptions, Time time, - RebalanceCallbackMetricsManager metrics) { + RebalanceCallbackMetricsManager metricsManager) { this.log = logContext.logger(getClass()); this.subscriptions = subscriptions; this.time = time; - this.metrics = metrics; + this.metricsManager = metricsManager; } public Exception invokePartitionsAssigned(final SortedSet assignedPartitions) { @@ -62,7 +62,7 @@ public Exception invokePartitionsAssigned(final SortedSet assign try { final long startMs = time.milliseconds(); listener.get().onPartitionsAssigned(assignedPartitions); - metrics.recordPartitionAssignLatency(time.milliseconds() - startMs); + metricsManager.recordPartitionAssignLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { @@ -88,7 +88,7 @@ public Exception invokePartitionsRevoked(final SortedSet revoked try { final long startMs = time.milliseconds(); listener.get().onPartitionsRevoked(revokedPartitions); - metrics.recordPartitionRevokeLatency(time.milliseconds() - startMs); + metricsManager.recordPartitionRevokeLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { @@ -114,7 +114,7 @@ public Exception invokePartitionsLost(final SortedSet lostPartit try { final long startMs = time.milliseconds(); listener.get().onPartitionsLost(lostPartitions); - metrics.recordPartitionLostLatency(time.milliseconds() - startMs); + metricsManager.recordPartitionLostLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 2e0e63f786e40..7afdff75cd2dd 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -110,7 +110,7 @@ public class HeartbeatRequestManager implements RequestManager { * sending heartbeat until the next poll. */ private final Timer pollTimer; - private final HeartbeatMetricsManager heartbeatMetricsManager; + private final HeartbeatMetricsManager metricsManager; private GroupMetadataUpdateEvent previousGroupMetadataUpdateEvent = null; @@ -134,7 +134,7 @@ public HeartbeatRequestManager( this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, retryBackoffMaxMs, maxPollIntervalMs); this.pollTimer = time.timer(maxPollIntervalMs); - this.heartbeatMetricsManager = new HeartbeatMetricsManager(metrics); + this.metricsManager = new HeartbeatMetricsManager(metrics); } // Visible for testing @@ -156,7 +156,7 @@ public HeartbeatRequestManager( this.membershipManager = membershipManager; this.backgroundEventHandler = backgroundEventHandler; this.pollTimer = timer; - this.heartbeatMetricsManager = new HeartbeatMetricsManager(metrics); + this.metricsManager = new HeartbeatMetricsManager(metrics); } /** @@ -252,7 +252,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final long curr NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(ignoreResponse); heartbeatRequestState.onSendAttempt(currentTimeMs); membershipManager.onHeartbeatRequestSent(); - heartbeatMetricsManager.recordHeartbeatSentMs(currentTimeMs); + metricsManager.recordHeartbeatSentMs(currentTimeMs); return request; } @@ -265,7 +265,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i else return request.whenComplete((response, exception) -> { if (response != null) { - heartbeatMetricsManager.recordRequestLatency(response.requestLatencyMs()); + metricsManager.recordRequestLatency(response.requestLatencyMs()); onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), request.handler().completionTimeMs()); } else { onFailure(exception, request.handler().completionTimeMs()); @@ -276,7 +276,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest(final boolean i private NetworkClientDelegate.UnsentRequest logResponse(final NetworkClientDelegate.UnsentRequest request) { return request.whenComplete((response, exception) -> { if (response != null) { - heartbeatMetricsManager.recordRequestLatency(response.requestLatencyMs()); + metricsManager.recordRequestLatency(response.requestLatencyMs()); Errors error = Errors.forCode(((ConsumerGroupHeartbeatResponse) response.responseBody()).data().errorCode()); if (error == Errors.NONE) From b17455ed3dfd25ed1806527402b771e5f4d27728 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Sun, 28 Jan 2024 21:10:38 -0800 Subject: [PATCH 21/28] resolving conflict --- .../consumer/internals/CommitRequestManager.java | 10 ++++++++-- .../internals/metrics/OffsetCommitMetricsManager.java | 7 +++++-- .../consumer/internals/CommitRequestManagerTest.java | 4 +++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java index 58d1ffd2d3c6d..e0fc4f264b875 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java @@ -470,7 +470,14 @@ public NetworkClientDelegate.UnsentRequest toUnsentRequest() { OffsetCommitRequest.Builder builder = new OffsetCommitRequest.Builder(data); - return buildRequestWithResponseHandling(builder); + return recordCommitLatency(buildRequestWithResponseHandling(builder)); + } + + private NetworkClientDelegate.UnsentRequest recordCommitLatency(NetworkClientDelegate.UnsentRequest request) { + request.future().whenComplete( + (r, e) -> metricsManager.recordRequestLatency(r) + ); + return request; } /** @@ -481,7 +488,6 @@ public NetworkClientDelegate.UnsentRequest toUnsentRequest() { */ @Override public void onResponse(final ClientResponse response) { - metricsManager.recordRequestLatency(response.requestLatencyMs()); long currentTimeMs = response.receivedTimeMs(); OffsetCommitResponse commitResponse = (OffsetCommitResponse) response.responseBody(); Set unauthorizedTopics = new HashSet<>(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java index 7aef981066820..4d6e192519e3f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; +import org.apache.kafka.clients.ClientResponse; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Avg; @@ -44,7 +45,9 @@ public OffsetCommitMetricsManager(Metrics metrics) { "The total number of commit calls"))); } - public void recordRequestLatency(long latencyMs) { - this.commitSensor.record(latencyMs); + public void recordRequestLatency(ClientResponse response) { + if (response == null) + return; + this.commitSensor.record(response.requestLatencyMs()); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java index 139d922efe2c6..a71e542de56b9 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java @@ -134,7 +134,6 @@ public void testPollEnsureManualCommitSent() { Map offsets = new HashMap<>(); offsets.put(new TopicPartition("t1", 0), new OffsetAndMetadata(0)); commitRequestManger.addOffsetCommitRequest(offsets, Optional.empty(), false); - assertPoll(1, commitRequestManger); } @Test @@ -154,6 +153,9 @@ public void testPollEnsureAutocommitSent() { 1, (short) 1, Errors.NONE))); + + assertEquals(0.03, (double) getMetric("commit-rate").metricValue(), 0.01); + assertEquals(1.0, getMetric("commit-total").metricValue()); } @Test From 6285909acdb3ee1a7cb943ee2a1d388d1a2de21c Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Sun, 28 Jan 2024 21:18:45 -0800 Subject: [PATCH 22/28] resolve conflict --- .../clients/consumer/internals/CommitRequestManager.java | 9 +-------- .../internals/metrics/OffsetCommitMetricsManager.java | 7 ++----- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java index ac71c034c126e..58d1ffd2d3c6d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java @@ -470,14 +470,7 @@ public NetworkClientDelegate.UnsentRequest toUnsentRequest() { OffsetCommitRequest.Builder builder = new OffsetCommitRequest.Builder(data); - return recordCommitLatency(buildRequestWithResponseHandling(builder)); - } - - private NetworkClientDelegate.UnsentRequest recordCommitLatency(NetworkClientDelegate.UnsentRequest request) { - request.future().whenComplete( - (r, e) -> metricsManager.recordRequestLatency(r) - ); - return request; + return buildRequestWithResponseHandling(builder); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java index 4d6e192519e3f..022a21b67895e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; -import org.apache.kafka.clients.ClientResponse; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Avg; @@ -45,9 +44,7 @@ public OffsetCommitMetricsManager(Metrics metrics) { "The total number of commit calls"))); } - public void recordRequestLatency(ClientResponse response) { - if (response == null) - return; - this.commitSensor.record(response.requestLatencyMs()); + public void recordRequestLatency(long responseLatencyMs) { + this.commitSensor.record(responseLatencyMs); } } From 1424ed9e96351f5ead0357fe77e2a657c328a3fb Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Mon, 29 Jan 2024 08:10:13 -0800 Subject: [PATCH 23/28] make naming consistent --- .../internals/ConsumerRebalanceListenerInvoker.java | 6 +++--- .../internals/metrics/RebalanceCallbackMetricsManager.java | 6 +++--- .../consumer/internals/ConsumerMetricsManagerTest.java | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java index 93d5c65bee539..dcdd303fe95d1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerRebalanceListenerInvoker.java @@ -62,7 +62,7 @@ public Exception invokePartitionsAssigned(final SortedSet assign try { final long startMs = time.milliseconds(); listener.get().onPartitionsAssigned(assignedPartitions); - metricsManager.recordPartitionAssignLatency(time.milliseconds() - startMs); + metricsManager.recordPartitionsAssignedLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { @@ -88,7 +88,7 @@ public Exception invokePartitionsRevoked(final SortedSet revoked try { final long startMs = time.milliseconds(); listener.get().onPartitionsRevoked(revokedPartitions); - metricsManager.recordPartitionRevokeLatency(time.milliseconds() - startMs); + metricsManager.recordPartitionsRevokedLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { @@ -114,7 +114,7 @@ public Exception invokePartitionsLost(final SortedSet lostPartit try { final long startMs = time.milliseconds(); listener.get().onPartitionsLost(lostPartitions); - metricsManager.recordPartitionLostLatency(time.milliseconds() - startMs); + metricsManager.recordPartitionsLostLatency(time.milliseconds() - startMs); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java index 19e7267805a0a..8a648e431f5d5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java @@ -60,15 +60,15 @@ public RebalanceCallbackMetricsManager(Metrics metrics, String grpMetricsPrefix) "The max time taken for a partition-lost rebalance listener callback"), new Max()); } - public void recordPartitionRevokeLatency(long latencyMs) { + public void recordPartitionsRevokedLatency(long latencyMs) { partitionRevokeCallbackSensor.record(latencyMs); } - public void recordPartitionAssignLatency(long latencyMs) { + public void recordPartitionsAssignedLatency(long latencyMs) { partitionAssignCallbackSensor.record(latencyMs); } - public void recordPartitionLostLatency(long latencyMs) { + public void recordPartitionsLostLatency(long latencyMs) { partitionLostCallbackSensor.record(latencyMs); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java index 5fad34987eeef..06300fbfa9439 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java @@ -47,9 +47,9 @@ public void testRebalanceCallbackMetrics() { assertNotNull(getMetric("partition-lost-latency-avg", groupName)); assertNotNull(getMetric("partition-lost-latency-max", groupName)); - rebalanceCallbackMetricsManager.recordPartitionAssignLatency(100); - rebalanceCallbackMetricsManager.recordPartitionRevokeLatency(101); - rebalanceCallbackMetricsManager.recordPartitionLostLatency(102); + rebalanceCallbackMetricsManager.recordPartitionsAssignedLatency(100); + rebalanceCallbackMetricsManager.recordPartitionsRevokedLatency(101); + rebalanceCallbackMetricsManager.recordPartitionsLostLatency(102); assertEquals(101d, getMetric("partition-revoked-latency-avg", groupName).metricValue()); assertEquals(101d, getMetric("partition-revoked-latency-max", groupName).metricValue()); From 7076ba50f7cf0d84df52eda00f37d4228711591b Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Mon, 29 Jan 2024 10:50:23 -0800 Subject: [PATCH 24/28] clean up --- .../internals/AsyncKafkaConsumer.java | 2 +- .../AbstractConsumerMetricsManager.java | 2 +- .../metrics/HeartbeatMetricsManager.java | 14 ++--- .../metrics/KafkaConsumerMetrics.java | 12 ++-- .../RebalanceCallbackMetricsManager.java | 12 ++-- .../internals/ConsumerMetricsManagerTest.java | 57 ++++++++++--------- 6 files changed, 50 insertions(+), 49 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java index 9198039a85a44..eaf21940912c9 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java @@ -539,7 +539,7 @@ private void process(final ConsumerRebalanceListenerCallbackNeededEvent event) { logContext, subscriptions, time, - new RebalanceCallbackMetricsManager(metrics, CONSUMER_METRIC_GROUP_PREFIX) + new RebalanceCallbackMetricsManager(metrics) ); ApiVersions apiVersions = new ApiVersions(); Supplier networkClientDelegateSupplier = () -> new NetworkClientDelegate( diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java index 1eafcfa95bfbc..9ccf2f8486725 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java @@ -37,7 +37,7 @@ *

        */ public abstract class AbstractConsumerMetricsManager { - protected final String metricGroupName; + private final String metricGroupName; final Metrics metrics; public enum MetricGroupSuffix { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java index 708d89ea2d510..e9b66ee53c732 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java @@ -27,18 +27,14 @@ public class HeartbeatMetricsManager extends AbstractConsumerMetricsManager { - public final Sensor heartbeatSensor; + private final Sensor heartbeatSensor; private long lastHeartbeatMs = -1L; public HeartbeatMetricsManager(Metrics metrics) { - this(metrics, CONSUMER_METRIC_GROUP_PREFIX); - } - - public HeartbeatMetricsManager(Metrics metrics, String prefix) { - super(metrics, prefix, MetricGroupSuffix.COORDINATOR); + super(metrics, CONSUMER_METRIC_GROUP_PREFIX, MetricGroupSuffix.COORDINATOR); heartbeatSensor = metrics.sensor("heartbeat-latency"); heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", - metricGroupName, + metricGroupName(), "The max time taken to receive a response to a heartbeat request"), new Max()); heartbeatSensor.add(createMeter(metrics, @@ -59,8 +55,8 @@ public HeartbeatMetricsManager(Metrics metrics, String prefix) { lastHeartbeat); } - public void recordHeartbeatSentMs(long lastHeartbeatMs) { - this.lastHeartbeatMs = lastHeartbeatMs; + public void recordHeartbeatSentMs(long timeMs) { + lastHeartbeatMs = timeMs; } public void recordRequestLatency(long requestLatencyMs) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java index ce32b9d97e02b..6badb9e35b95e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java @@ -48,22 +48,22 @@ public KafkaConsumerMetrics(Metrics metrics, String metricGrpPrefix) { return TimeUnit.SECONDS.convert(now - lastPollMs, TimeUnit.MILLISECONDS); }; this.lastPollMetricName = metrics.metricName("last-poll-seconds-ago", - metricGroupName, "The number of seconds since the last poll() invocation."); + metricGroupName(), "The number of seconds since the last poll() invocation."); metrics.addMetric(lastPollMetricName, lastPoll); this.timeBetweenPollSensor = metrics.sensor("time-between-poll"); this.timeBetweenPollSensor.add(metrics.metricName("time-between-poll-avg", - metricGroupName, + metricGroupName(), "The average delay between invocations of poll() in milliseconds."), new Avg()); this.timeBetweenPollSensor.add(metrics.metricName("time-between-poll-max", - metricGroupName, + metricGroupName(), "The max delay between invocations of poll() in milliseconds."), new Max()); this.pollIdleSensor = metrics.sensor("poll-idle-ratio-avg"); this.pollIdleSensor.add(metrics.metricName("poll-idle-ratio-avg", - metricGroupName, + metricGroupName(), "The average fraction of time the consumer's poll() is idle as opposed to waiting for the user code to process records."), new Avg()); @@ -71,7 +71,7 @@ public KafkaConsumerMetrics(Metrics metrics, String metricGrpPrefix) { this.commitSyncSensor.add( metrics.metricName( "commit-sync-time-ns-total", - metricGroupName, + metricGroupName(), "The total time the consumer has spent in commitSync in nanoseconds" ), new CumulativeSum() @@ -81,7 +81,7 @@ public KafkaConsumerMetrics(Metrics metrics, String metricGrpPrefix) { this.committedSensor.add( metrics.metricName( "committed-time-ns-total", - metricGroupName, + metricGroupName(), "The total time the consumer has spent in committed in nanoseconds" ), new CumulativeSum() diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java index 8a648e431f5d5..85bcaf9404ec8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java @@ -37,26 +37,26 @@ public RebalanceCallbackMetricsManager(Metrics metrics, String grpMetricsPrefix) super(metrics, grpMetricsPrefix, COORDINATOR); partitionRevokeCallbackSensor = metrics.sensor("partition-revoked-latency"); partitionRevokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-avg", - metricGroupName, + metricGroupName(), "The average time taken for a partition-revoked rebalance listener callback"), new Avg()); partitionRevokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-max", - metricGroupName, + metricGroupName(), "The max time taken for a partition-revoked rebalance listener callback"), new Max()); partitionAssignCallbackSensor = metrics.sensor("partition-assigned-latency"); partitionAssignCallbackSensor.add(metrics.metricName("partition-assigned-latency-avg", - metricGroupName, + metricGroupName(), "The average time taken for a partition-assigned rebalance listener callback"), new Avg()); partitionAssignCallbackSensor.add(metrics.metricName("partition-assigned-latency-max", - metricGroupName, + metricGroupName(), "The max time taken for a partition-assigned rebalance listener callback"), new Max()); partitionLostCallbackSensor = metrics.sensor("partition-lost-latency"); partitionLostCallbackSensor.add(metrics.metricName("partition-lost-latency-avg", - metricGroupName, + metricGroupName(), "The average time taken for a partition-lost rebalance listener callback"), new Avg()); partitionLostCallbackSensor.add(metrics.metricName("partition-lost-latency-max", - metricGroupName, + metricGroupName(), "The max time taken for a partition-lost rebalance listener callback"), new Max()); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java index 06300fbfa9439..706b5dee137ca 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java @@ -16,8 +16,9 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.consumer.internals.metrics.AbstractConsumerMetricsManager; import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetricsManager; +import org.apache.kafka.clients.consumer.internals.metrics.KafkaConsumerMetrics; +import org.apache.kafka.clients.consumer.internals.metrics.OffsetCommitMetricsManager; import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; import org.apache.kafka.common.metrics.KafkaMetric; import org.apache.kafka.common.metrics.Metrics; @@ -25,11 +26,12 @@ import org.apache.kafka.common.utils.Time; import org.junit.jupiter.api.Test; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; public class ConsumerMetricsManagerTest { private Time time = new MockTime(); @@ -68,37 +70,40 @@ public void testHeartbeatMetrics() { assertNotNull(getMetric("heartbeat-rate", groupName)); assertNotNull(getMetric("heartbeat-total", groupName)); - heartbeatMetricsManager.heartbeatSensor.record(100); - heartbeatMetricsManager.heartbeatSensor.record(100); - heartbeatMetricsManager.heartbeatSensor.record(100); + heartbeatMetricsManager.recordHeartbeatSentMs(time.milliseconds()); + heartbeatMetricsManager.recordRequestLatency(100); + heartbeatMetricsManager.recordRequestLatency(103); + heartbeatMetricsManager.recordRequestLatency(102); - assertEquals(100d, getMetric("heartbeat-response-time-max", groupName).metricValue()); + assertEquals(103d, getMetric("heartbeat-response-time-max", groupName).metricValue()); assertEquals(0.1d, (double) getMetric("heartbeat-rate", groupName).metricValue(), 0.01d); assertEquals(3d, getMetric("heartbeat-total", groupName).metricValue()); + + // Randomly sleep 1-10 seconds + Random rand = new Random(); + int randomSleepS = rand.nextInt(10) + 1; + time.sleep(TimeUnit.SECONDS.toMillis(randomSleepS)); + assertEquals((double) randomSleepS, getMetric("last-heartbeat-seconds-ago", groupName).metricValue()); } @Test - public void testMetricsGroupName() { - // consumer-coordinator-metrics - NoopConsumerMetrics coordinatorMetrics = new NoopConsumerMetrics(metrics, "customCoordinatorPrefix", AbstractConsumerMetricsManager.MetricGroupSuffix.COORDINATOR); - assertEquals("customCoordinatorPrefix" + COORDINATOR_METRICS_SUFFIX, coordinatorMetrics.metricGroupName()); - - // consumer-metrics - NoopConsumerMetrics consumerMetrics = new NoopConsumerMetrics(metrics, "customConsumerPrefix", AbstractConsumerMetricsManager.MetricGroupSuffix.CONSUMER); - assertEquals("customConsumerPrefix" + CONSUMER_METRICS_SUFFIX, consumerMetrics.metricGroupName()); - - assertThrows(IllegalArgumentException.class, () -> new NoopConsumerMetrics(metrics, - null, - AbstractConsumerMetricsManager.MetricGroupSuffix.CONSUMER)); + public void testDefaultMetricsGroupName() { + final String coordinatorMetricsGroupName = "consumer-coordinator-metrics"; + HeartbeatMetricsManager heartbeatMetrics = new HeartbeatMetricsManager(metrics); + assertEquals(coordinatorMetricsGroupName, heartbeatMetrics.metricGroupName()); + + OffsetCommitMetricsManager commitMetrics = new OffsetCommitMetricsManager(metrics); + assertEquals(coordinatorMetricsGroupName, commitMetrics.metricGroupName()); + + RebalanceCallbackMetricsManager rebalanceCallbackMetrics = new RebalanceCallbackMetricsManager(metrics); + assertEquals(coordinatorMetricsGroupName, rebalanceCallbackMetrics.metricGroupName()); + + final String consumerMetricsGroupName = "consumer-metrics"; + KafkaConsumerMetrics consumerMetrics = new KafkaConsumerMetrics(metrics, CONSUMER_METRIC_GROUP_PREFIX); + assertEquals(consumerMetricsGroupName, consumerMetrics.metricGroupName()); } private KafkaMetric getMetric(String metricName, String groupName) { return metrics.metrics().get(metrics.metricName(metricName, groupName)); } - - static class NoopConsumerMetrics extends AbstractConsumerMetricsManager { - public NoopConsumerMetrics(Metrics metrics, String prefix, MetricGroupSuffix suffix) { - super(metrics, prefix, suffix); - } - } } From 4eddba438cba94f4c84602605cbd3f7feee51f29 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Mon, 29 Jan 2024 10:58:47 -0800 Subject: [PATCH 25/28] Update HeartbeatRequestManagerTest.java --- .../consumer/internals/HeartbeatRequestManagerTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index 33d54ab66389d..cb870120f5442 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -102,7 +102,7 @@ public class HeartbeatRequestManagerTest { private final int memberEpoch = 1; private BackgroundEventHandler backgroundEventHandler; private BlockingQueue backgroundEventQueue; - private Metrics metrics = new Metrics(); + private Metrics metrics; @BeforeEach public void setUp() { @@ -121,6 +121,7 @@ private void setUp(Optional groupInfo) { subscriptions = testBuilder.subscriptions; membershipManager = testBuilder.membershipManager.orElseThrow(IllegalStateException::new); metadata = testBuilder.metadata; + metrics = new Metrics(time); when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); } @@ -570,18 +571,19 @@ public void testPollTimerExpiration() { @Test public void testHeartbeatMetrics() { + // setup coordinatorRequestManager = mock(CoordinatorRequestManager.class); membershipManager = mock(MembershipManager.class); heartbeatState = mock(HeartbeatRequestManager.HeartbeatState.class); time = new MockTime(); metrics = new Metrics(time); - heartbeatRequestState = spy(new HeartbeatRequestManager.HeartbeatRequestState( + heartbeatRequestState = new HeartbeatRequestManager.HeartbeatRequestState( new LogContext(), time, 0, // This initial interval should be 0 to ensure heartbeat on the clock retryBackoffMs, retryBackoffMaxMs, - 0)); + 0); backgroundEventHandler = mock(BackgroundEventHandler.class); heartbeatRequestManager = createHeartbeatRequestManager( coordinatorRequestManager, @@ -597,6 +599,7 @@ public void testHeartbeatMetrics() { assertNotNull(getMetric("heartbeat-total")); assertNotNull(getMetric("last-heartbeat-seconds-ago")); + // test poll assertHeartbeat(heartbeatRequestManager, 0); time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(1.0, getMetric("heartbeat-total").metricValue()); From 9c86077775e86cd0e17a68f68fc1dd39f616e2fb Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Mon, 29 Jan 2024 11:24:18 -0800 Subject: [PATCH 26/28] privatize metric field --- .../metrics/AbstractConsumerMetricsManager.java | 6 +++++- .../internals/metrics/KafkaConsumerMetrics.java | 10 +++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java index 9ccf2f8486725..fc980ce1a0fc8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java @@ -38,7 +38,7 @@ */ public abstract class AbstractConsumerMetricsManager { private final String metricGroupName; - final Metrics metrics; + private final Metrics metrics; public enum MetricGroupSuffix { COORDINATOR(COORDINATOR_METRICS_SUFFIX), @@ -69,6 +69,10 @@ public String metricGroupName() { return metricGroupName; } + public Metrics metrics() { + return metrics; + } + public Meter createMeter(Metrics metrics, String baseName, String descriptiveName) { return new Meter(new WindowedCount(), metrics.metricName(baseName + "-rate", diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java index 6badb9e35b95e..6c62a066091da 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java @@ -111,10 +111,10 @@ public void recordCommitted(long duration) { @Override public void close() { - metrics.removeMetric(lastPollMetricName); - metrics.removeSensor(timeBetweenPollSensor.name()); - metrics.removeSensor(pollIdleSensor.name()); - metrics.removeSensor(commitSyncSensor.name()); - metrics.removeSensor(committedSensor.name()); + metrics().removeMetric(lastPollMetricName); + metrics().removeSensor(timeBetweenPollSensor.name()); + metrics().removeSensor(pollIdleSensor.name()); + metrics().removeSensor(commitSyncSensor.name()); + metrics().removeSensor(committedSensor.name()); } } From 8964a8d2be4d8cea0cca39c371691b0c6aed22c1 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Wed, 31 Jan 2024 09:15:00 -0800 Subject: [PATCH 27/28] PR comments --- .../internals/CommitRequestManager.java | 2 +- .../internals/HeartbeatRequestManager.java | 7 +++++-- .../HeartbeatRequestManagerTest.java | 19 +++++++++++-------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java index 1751d98c75315..b81dc03a9bfec 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java @@ -82,7 +82,7 @@ public class CommitRequestManager implements RequestManager, MemberStateListener private final boolean throwOnFetchStableOffsetUnsupported; final PendingRequests pendingRequests; private boolean closing = false; - private OffsetCommitMetricsManager metricsManager; + private final OffsetCommitMetricsManager metricsManager; /** * Latest member ID and epoch received via the {@link #onMemberEpochUpdated(Optional, Optional)}, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 7afdff75cd2dd..03e11ddfa02b4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -110,10 +110,13 @@ public class HeartbeatRequestManager implements RequestManager { * sending heartbeat until the next poll. */ private final Timer pollTimer; - private final HeartbeatMetricsManager metricsManager; - private GroupMetadataUpdateEvent previousGroupMetadataUpdateEvent = null; + /** + * Holding the heartbeat sensor to measure heartbeat timing and response latency + */ + private final HeartbeatMetricsManager metricsManager; + public HeartbeatRequestManager( final LogContext logContext, final Time time, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index cb870120f5442..bcc66c1f69970 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -56,6 +56,7 @@ import java.util.List; import java.util.Optional; import java.util.Properties; +import java.util.Random; import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; @@ -65,8 +66,6 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_REMOTE_ASSIGNOR; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MAX_MS; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MS; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -86,7 +85,7 @@ public class HeartbeatRequestManagerTest { private int maxPollIntervalMs = DEFAULT_MAX_POLL_INTERVAL_MS; private long retryBackoffMaxMs = DEFAULT_RETRY_BACKOFF_MAX_MS; private static final String DEFAULT_GROUP_ID = "groupId"; - private String metricGroupPrefix = CONSUMER_METRIC_GROUP_PREFIX; + private String consumerMetricsGroupName = "consumer-coordinator-metrics"; private ConsumerTestBuilder testBuilder; private Time time; @@ -601,15 +600,19 @@ public void testHeartbeatMetrics() { // test poll assertHeartbeat(heartbeatRequestManager, 0); - time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); + time.sleep(heartbeatIntervalMs); assertEquals(1.0, getMetric("heartbeat-total").metricValue()); assertEquals((double) TimeUnit.MILLISECONDS.toSeconds(heartbeatIntervalMs), getMetric("last-heartbeat-seconds-ago").metricValue()); - assertHeartbeat(heartbeatRequestManager, DEFAULT_HEARTBEAT_INTERVAL_MS); + assertHeartbeat(heartbeatRequestManager, heartbeatIntervalMs); assertEquals(0.06d, (double) getMetric("heartbeat-rate").metricValue(), 0.005d); assertEquals(2.0, getMetric("heartbeat-total").metricValue()); - time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); - assertEquals((double) TimeUnit.MILLISECONDS.toSeconds(heartbeatIntervalMs), getMetric("last-heartbeat-seconds-ago").metricValue()); + + // Randomly sleep for some time + Random rand = new Random(); + int randomSleepS = rand.nextInt(11); + time.sleep(randomSleepS * 1000); + assertEquals((double) randomSleepS, getMetric("last-heartbeat-seconds-ago").metricValue()); } private void assertHeartbeat(HeartbeatRequestManager hrm, int nextPollMs) { @@ -722,7 +725,7 @@ private ConsumerConfig config() { } private KafkaMetric getMetric(final String name) { - return metrics.metrics().get(metrics.metricName(name, metricGroupPrefix + COORDINATOR_METRICS_SUFFIX)); + return metrics.metrics().get(metrics.metricName(name, consumerMetricsGroupName)); } private HeartbeatRequestManager createHeartbeatRequestManager( From 22fcb3915e4344f1fd6c4aafa52b423c5a3aec88 Mon Sep 17 00:00:00 2001 From: Philip Nee Date: Wed, 31 Jan 2024 11:04:28 -0800 Subject: [PATCH 28/28] clean up according to PR comments --- .../internals/CommitRequestManager.java | 2 +- .../AbstractConsumerMetricsManager.java | 94 --------------- .../metrics/HeartbeatMetricsManager.java | 40 ++++--- .../metrics/KafkaConsumerMetrics.java | 30 ++--- .../metrics/OffsetCommitMetricsManager.java | 36 ++++-- .../RebalanceCallbackMetricsManager.java | 55 +++++---- .../internals/CommitRequestManagerTest.java | 6 +- .../internals/ConsumerMetricsManagerTest.java | 109 ------------------ .../HeartbeatRequestManagerTest.java | 4 +- .../metrics/HeartbeatMetricsManagerTest.java | 62 ++++++++++ .../OffsetCommitMetricsManagerTest.java | 55 +++++++++ .../RebalanceCallbackMetricsManagerTest.java | 52 +++++++++ 12 files changed, 275 insertions(+), 270 deletions(-) delete mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java delete mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java create mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManagerTest.java create mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManagerTest.java create mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManagerTest.java diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java index b81dc03a9bfec..2ed6d6ca7da17 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java @@ -73,6 +73,7 @@ public class CommitRequestManager implements RequestManager, MemberStateListener private final Optional autoCommitState; private final CoordinatorRequestManager coordinatorRequestManager; private final OffsetCommitCallbackInvoker offsetCommitCallbackInvoker; + private final OffsetCommitMetricsManager metricsManager; private final long retryBackoffMs; private final String groupId; private final Optional groupInstanceId; @@ -82,7 +83,6 @@ public class CommitRequestManager implements RequestManager, MemberStateListener private final boolean throwOnFetchStableOffsetUnsupported; final PendingRequests pendingRequests; private boolean closing = false; - private final OffsetCommitMetricsManager metricsManager; /** * Latest member ID and epoch received via the {@link #onMemberEpochUpdated(Optional, Optional)}, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java deleted file mode 100644 index fc980ce1a0fc8..0000000000000 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/AbstractConsumerMetricsManager.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals.metrics; - -import org.apache.kafka.common.metrics.Measurable; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.stats.Meter; -import org.apache.kafka.common.metrics.stats.WindowedCount; - -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; - -/** - * Base class for different consumer metrics to extend. This class helps to construct the logical group name from the - * given prefix and suffix, and provides a few common utilities. - * - *

        - * The suffix can be one of the following: - *

          - *
        • -coordinator-metrics: {@link MetricGroupSuffix#COORDINATOR}
          • - *
        • -metrics: {@link MetricGroupSuffix#CONSUMER}
          • - *
        - *

        - */ -public abstract class AbstractConsumerMetricsManager { - private final String metricGroupName; - private final Metrics metrics; - - public enum MetricGroupSuffix { - COORDINATOR(COORDINATOR_METRICS_SUFFIX), - CONSUMER(CONSUMER_METRICS_SUFFIX); - - private final String suffix; - - MetricGroupSuffix(String suffix) { - this.suffix = suffix; - } - - @Override - public String toString() { - return suffix; - } - } - - public AbstractConsumerMetricsManager(Metrics metrics, String prefix, MetricGroupSuffix suffix) { - if (suffix == null) - throw new IllegalArgumentException("metric group suffix cannot be null"); - if (prefix == null) - throw new IllegalArgumentException("metric group prefix cannot be null"); - this.metrics = metrics; - metricGroupName = prefix + suffix; - } - - public String metricGroupName() { - return metricGroupName; - } - - public Metrics metrics() { - return metrics; - } - - public Meter createMeter(Metrics metrics, String baseName, String descriptiveName) { - return new Meter(new WindowedCount(), - metrics.metricName(baseName + "-rate", - metricGroupName, - String.format("The number of %s per second", descriptiveName)), - metrics.metricName(baseName + "-total", - metricGroupName, - String.format("The total number of %s", descriptiveName))); - } - - public void addMetric(String metricName, - String description, - Measurable measurable) { - metrics.addMetric( - metrics.metricName(metricName, metricGroupName, description), - measurable); - } -} - diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java index e9b66ee53c732..a5654dfc85532 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManager.java @@ -16,30 +16,42 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; +import org.apache.kafka.common.MetricName; import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Max; +import org.apache.kafka.common.metrics.stats.Meter; +import org.apache.kafka.common.metrics.stats.WindowedCount; import java.util.concurrent.TimeUnit; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; -public class HeartbeatMetricsManager extends AbstractConsumerMetricsManager { - +public class HeartbeatMetricsManager { + // MetricName visible for testing + final MetricName heartbeatResponseTimeMax; + final MetricName heartbeatRate; + final MetricName heartbeatTotal; + final MetricName lastHeartbeatSecondsAgo; private final Sensor heartbeatSensor; private long lastHeartbeatMs = -1L; public HeartbeatMetricsManager(Metrics metrics) { - super(metrics, CONSUMER_METRIC_GROUP_PREFIX, MetricGroupSuffix.COORDINATOR); + final String metricGroupName = CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX; heartbeatSensor = metrics.sensor("heartbeat-latency"); - heartbeatSensor.add(metrics.metricName("heartbeat-response-time-max", - metricGroupName(), - "The max time taken to receive a response to a heartbeat request"), - new Max()); - heartbeatSensor.add(createMeter(metrics, - "heartbeat", - "heartbeats")); + heartbeatResponseTimeMax = metrics.metricName("heartbeat-response-time-max", + metricGroupName, + "The max time taken to receive a response to a heartbeat request"); + heartbeatSensor.add(heartbeatResponseTimeMax, new Max()); + + // windowed meters + heartbeatRate = metrics.metricName("heartbeat-rate", metricGroupName, "The number of heartbeats per second"); + heartbeatTotal = metrics.metricName("heartbeat-total", metricGroupName, "The total number of heartbeats"); + heartbeatSensor.add(new Meter(new WindowedCount(), + heartbeatRate, + heartbeatTotal)); Measurable lastHeartbeat = (config, now) -> { final long lastHeartbeatSend = lastHeartbeatMs; @@ -49,10 +61,10 @@ public HeartbeatMetricsManager(Metrics metrics) { else return TimeUnit.SECONDS.convert(now - lastHeartbeatSend, TimeUnit.MILLISECONDS); }; - addMetric( - "last-heartbeat-seconds-ago", - "The number of seconds since the last coordinator heartbeat was sent", - lastHeartbeat); + lastHeartbeatSecondsAgo = metrics.metricName("last-heartbeat-seconds-ago", + metricGroupName, + "The number of seconds since the last coordinator heartbeat was sent"); + metrics.addMetric(lastHeartbeatSecondsAgo, lastHeartbeat); } public void recordHeartbeatSentMs(long timeMs) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java index 6c62a066091da..52502e714a947 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/KafkaConsumerMetrics.java @@ -26,9 +26,10 @@ import java.util.concurrent.TimeUnit; -import static org.apache.kafka.clients.consumer.internals.metrics.AbstractConsumerMetricsManager.MetricGroupSuffix.CONSUMER; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRICS_SUFFIX; -public class KafkaConsumerMetrics extends AbstractConsumerMetricsManager implements AutoCloseable { +public class KafkaConsumerMetrics implements AutoCloseable { + private final Metrics metrics; private final MetricName lastPollMetricName; private final Sensor timeBetweenPollSensor; private final Sensor pollIdleSensor; @@ -39,7 +40,8 @@ public class KafkaConsumerMetrics extends AbstractConsumerMetricsManager impleme private long timeSinceLastPollMs; public KafkaConsumerMetrics(Metrics metrics, String metricGrpPrefix) { - super(metrics, metricGrpPrefix, CONSUMER); + this.metrics = metrics; + final String metricGroupName = metricGrpPrefix + CONSUMER_METRICS_SUFFIX; Measurable lastPoll = (mConfig, now) -> { if (lastPollMs == 0L) // if no poll is ever triggered, just return -1. @@ -48,22 +50,22 @@ public KafkaConsumerMetrics(Metrics metrics, String metricGrpPrefix) { return TimeUnit.SECONDS.convert(now - lastPollMs, TimeUnit.MILLISECONDS); }; this.lastPollMetricName = metrics.metricName("last-poll-seconds-ago", - metricGroupName(), "The number of seconds since the last poll() invocation."); + metricGroupName, "The number of seconds since the last poll() invocation."); metrics.addMetric(lastPollMetricName, lastPoll); this.timeBetweenPollSensor = metrics.sensor("time-between-poll"); this.timeBetweenPollSensor.add(metrics.metricName("time-between-poll-avg", - metricGroupName(), + metricGroupName, "The average delay between invocations of poll() in milliseconds."), new Avg()); this.timeBetweenPollSensor.add(metrics.metricName("time-between-poll-max", - metricGroupName(), + metricGroupName, "The max delay between invocations of poll() in milliseconds."), new Max()); this.pollIdleSensor = metrics.sensor("poll-idle-ratio-avg"); this.pollIdleSensor.add(metrics.metricName("poll-idle-ratio-avg", - metricGroupName(), + metricGroupName, "The average fraction of time the consumer's poll() is idle as opposed to waiting for the user code to process records."), new Avg()); @@ -71,7 +73,7 @@ public KafkaConsumerMetrics(Metrics metrics, String metricGrpPrefix) { this.commitSyncSensor.add( metrics.metricName( "commit-sync-time-ns-total", - metricGroupName(), + metricGroupName, "The total time the consumer has spent in commitSync in nanoseconds" ), new CumulativeSum() @@ -81,7 +83,7 @@ public KafkaConsumerMetrics(Metrics metrics, String metricGrpPrefix) { this.committedSensor.add( metrics.metricName( "committed-time-ns-total", - metricGroupName(), + metricGroupName, "The total time the consumer has spent in committed in nanoseconds" ), new CumulativeSum() @@ -111,10 +113,10 @@ public void recordCommitted(long duration) { @Override public void close() { - metrics().removeMetric(lastPollMetricName); - metrics().removeSensor(timeBetweenPollSensor.name()); - metrics().removeSensor(pollIdleSensor.name()); - metrics().removeSensor(commitSyncSensor.name()); - metrics().removeSensor(committedSensor.name()); + metrics.removeMetric(lastPollMetricName); + metrics.removeSensor(timeBetweenPollSensor.name()); + metrics.removeSensor(pollIdleSensor.name()); + metrics.removeSensor(commitSyncSensor.name()); + metrics.removeSensor(committedSensor.name()); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java index 022a21b67895e..d700299ef1801 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManager.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; +import org.apache.kafka.common.MetricName; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Avg; @@ -24,24 +25,35 @@ import org.apache.kafka.common.metrics.stats.WindowedCount; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; -public class OffsetCommitMetricsManager extends AbstractConsumerMetricsManager { +public class OffsetCommitMetricsManager { + final MetricName commitLatencyAvg; + final MetricName commitLatencyMax; + final MetricName commitRate; + final MetricName commitTotal; private final Sensor commitSensor; public OffsetCommitMetricsManager(Metrics metrics) { - super(metrics, CONSUMER_METRIC_GROUP_PREFIX, MetricGroupSuffix.COORDINATOR); + final String metricGroupName = CONSUMER_METRIC_GROUP_PREFIX + COORDINATOR_METRICS_SUFFIX; commitSensor = metrics.sensor("commit-latency"); - commitSensor.add(metrics.metricName("commit-latency-avg", - metricGroupName(), - "The average time taken for a commit request"), new Avg()); - commitSensor.add(metrics.metricName("commit-latency-max", - metricGroupName(), - "The max time taken for a commit request"), new Max()); + commitLatencyAvg = metrics.metricName("commit-latency-avg", + metricGroupName, + "The average time taken for a commit request"); + commitSensor.add(commitLatencyAvg, new Avg()); + commitLatencyMax = metrics.metricName("commit-latency-max", + metricGroupName, + "The max time taken for a commit request"); + commitSensor.add(commitLatencyMax, new Max()); + commitRate = metrics.metricName("commit-rate", + metricGroupName, + "The number of commit calls per second"); + commitTotal = metrics.metricName("commit-total", + metricGroupName, + "The total number of commit calls"); commitSensor.add(new Meter(new WindowedCount(), - metrics.metricName("commit-rate", metricGroupName(), - "The number of commit calls per second"), - metrics.metricName("commit-total", metricGroupName(), - "The total number of commit calls"))); + commitRate, + commitTotal)); } public void recordRequestLatency(long responseLatencyMs) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java index 85bcaf9404ec8..f70b891864f1c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManager.java @@ -16,15 +16,22 @@ */ package org.apache.kafka.clients.consumer.internals.metrics; +import org.apache.kafka.common.MetricName; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Avg; import org.apache.kafka.common.metrics.stats.Max; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; -import static org.apache.kafka.clients.consumer.internals.metrics.AbstractConsumerMetricsManager.MetricGroupSuffix.COORDINATOR; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; -public class RebalanceCallbackMetricsManager extends AbstractConsumerMetricsManager { +public class RebalanceCallbackMetricsManager { + final MetricName partitionRevokeLatencyAvg; + final MetricName partitionAssignLatencyAvg; + final MetricName partitionLostLatencyAvg; + final MetricName partitionRevokeLatencyMax; + final MetricName partitionAssignLatencyMax; + final MetricName partitionLostLatencyMax; private final Sensor partitionRevokeCallbackSensor; private final Sensor partitionAssignCallbackSensor; private final Sensor partitionLostCallbackSensor; @@ -34,30 +41,36 @@ public RebalanceCallbackMetricsManager(Metrics metrics) { } public RebalanceCallbackMetricsManager(Metrics metrics, String grpMetricsPrefix) { - super(metrics, grpMetricsPrefix, COORDINATOR); + final String metricGroupName = grpMetricsPrefix + COORDINATOR_METRICS_SUFFIX; partitionRevokeCallbackSensor = metrics.sensor("partition-revoked-latency"); - partitionRevokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-avg", - metricGroupName(), - "The average time taken for a partition-revoked rebalance listener callback"), new Avg()); - partitionRevokeCallbackSensor.add(metrics.metricName("partition-revoked-latency-max", - metricGroupName(), - "The max time taken for a partition-revoked rebalance listener callback"), new Max()); + partitionRevokeLatencyAvg = metrics.metricName("partition-revoked-latency-avg", + metricGroupName, + "The average time taken for a partition-revoked rebalance listener callback"); + partitionRevokeCallbackSensor.add(partitionRevokeLatencyAvg, new Avg()); + partitionRevokeLatencyMax = metrics.metricName("partition-revoked-latency-max", + metricGroupName, + "The max time taken for a partition-revoked rebalance listener callback"); + partitionRevokeCallbackSensor.add(partitionRevokeLatencyMax, new Max()); partitionAssignCallbackSensor = metrics.sensor("partition-assigned-latency"); - partitionAssignCallbackSensor.add(metrics.metricName("partition-assigned-latency-avg", - metricGroupName(), - "The average time taken for a partition-assigned rebalance listener callback"), new Avg()); - partitionAssignCallbackSensor.add(metrics.metricName("partition-assigned-latency-max", - metricGroupName(), - "The max time taken for a partition-assigned rebalance listener callback"), new Max()); + partitionAssignLatencyAvg = metrics.metricName("partition-assigned-latency-avg", + metricGroupName, + "The average time taken for a partition-assigned rebalance listener callback"); + partitionAssignCallbackSensor.add(partitionAssignLatencyAvg, new Avg()); + partitionAssignLatencyMax = metrics.metricName("partition-assigned-latency-max", + metricGroupName, + "The max time taken for a partition-assigned rebalance listener callback"); + partitionAssignCallbackSensor.add(partitionAssignLatencyMax, new Max()); partitionLostCallbackSensor = metrics.sensor("partition-lost-latency"); - partitionLostCallbackSensor.add(metrics.metricName("partition-lost-latency-avg", - metricGroupName(), - "The average time taken for a partition-lost rebalance listener callback"), new Avg()); - partitionLostCallbackSensor.add(metrics.metricName("partition-lost-latency-max", - metricGroupName(), - "The max time taken for a partition-lost rebalance listener callback"), new Max()); + partitionLostLatencyAvg = metrics.metricName("partition-lost-latency-avg", + metricGroupName, + "The average time taken for a partition-lost rebalance listener callback"); + partitionLostCallbackSensor.add(partitionLostLatencyAvg, new Avg()); + partitionLostLatencyMax = metrics.metricName("partition-lost-latency-max", + metricGroupName, + "The max time taken for a partition-lost rebalance listener callback"); + partitionLostCallbackSensor.add(partitionLostLatencyMax, new Max()); } public void recordPartitionsRevokedLatency(long latencyMs) { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java index c32f63857a5d1..15e1e0b4fb4bc 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java @@ -75,7 +75,6 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_GROUP_ID; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_GROUP_INSTANCE_ID; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.COORDINATOR_METRICS_SUFFIX; import static org.apache.kafka.test.TestUtils.assertFutureThrows; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -94,7 +93,7 @@ public class CommitRequestManagerTest { private long retryBackoffMs = 100; private long retryBackoffMaxMs = 1000; private String consumerMetricGroupPrefix = CONSUMER_METRIC_GROUP_PREFIX; - private String consumerMetricGroupName = consumerMetricGroupPrefix + COORDINATOR_METRICS_SUFFIX; + private static final String CONSUMER_COORDINATOR_METRICS = "consumer-coordinator-metrics"; private Node mockedNode = new Node(1, "host1", 9092); private SubscriptionState subscriptionState; private LogContext logContext; @@ -139,6 +138,7 @@ public void testPollEnsureManualCommitSent() { Map offsets = new HashMap<>(); offsets.put(new TopicPartition("t1", 0), new OffsetAndMetadata(0)); commitRequestManger.addOffsetCommitRequest(offsets, Optional.empty(), false); + assertPoll(1, commitRequestManger); } @Test @@ -1247,6 +1247,6 @@ private ClientResponse buildOffsetFetchClientResponse( private KafkaMetric getMetric(String name) { return metrics.metrics().get(metrics.metricName( name, - consumerMetricGroupName)); + CONSUMER_COORDINATOR_METRICS)); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java deleted file mode 100644 index 706b5dee137ca..0000000000000 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetricsManagerTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals; - -import org.apache.kafka.clients.consumer.internals.metrics.HeartbeatMetricsManager; -import org.apache.kafka.clients.consumer.internals.metrics.KafkaConsumerMetrics; -import org.apache.kafka.clients.consumer.internals.metrics.OffsetCommitMetricsManager; -import org.apache.kafka.clients.consumer.internals.metrics.RebalanceCallbackMetricsManager; -import org.apache.kafka.common.metrics.KafkaMetric; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.common.utils.Time; -import org.junit.jupiter.api.Test; - -import java.util.Random; -import java.util.concurrent.TimeUnit; - -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -public class ConsumerMetricsManagerTest { - private Time time = new MockTime(); - private Metrics metrics = new Metrics(time); - - @Test - public void testRebalanceCallbackMetrics() { - // MBean: kafka.consumer:type=consumer-coordinator-metrics,client-id={clientId} - RebalanceCallbackMetricsManager rebalanceCallbackMetricsManager = new RebalanceCallbackMetricsManager(metrics); - String groupName = "consumer-coordinator-metrics"; - assertNotNull(getMetric("partition-revoked-latency-avg", groupName)); - assertNotNull(getMetric("partition-revoked-latency-max", groupName)); - assertNotNull(getMetric("partition-assigned-latency-avg", groupName)); - assertNotNull(getMetric("partition-assigned-latency-max", groupName)); - assertNotNull(getMetric("partition-lost-latency-avg", groupName)); - assertNotNull(getMetric("partition-lost-latency-max", groupName)); - - rebalanceCallbackMetricsManager.recordPartitionsAssignedLatency(100); - rebalanceCallbackMetricsManager.recordPartitionsRevokedLatency(101); - rebalanceCallbackMetricsManager.recordPartitionsLostLatency(102); - - assertEquals(101d, getMetric("partition-revoked-latency-avg", groupName).metricValue()); - assertEquals(101d, getMetric("partition-revoked-latency-max", groupName).metricValue()); - assertEquals(100d, getMetric("partition-assigned-latency-avg", groupName).metricValue()); - assertEquals(100d, getMetric("partition-assigned-latency-max", groupName).metricValue()); - assertEquals(102d, getMetric("partition-lost-latency-avg", groupName).metricValue()); - assertEquals(102d, getMetric("partition-lost-latency-max", groupName).metricValue()); - } - - @Test - public void testHeartbeatMetrics() { - // MBean: kafka.consumer:type=consumer-coordinator-metrics,client-id={clientId} - HeartbeatMetricsManager heartbeatMetricsManager = new HeartbeatMetricsManager(metrics); - String groupName = "consumer-coordinator-metrics"; - assertNotNull(getMetric("heartbeat-response-time-max", groupName)); - assertNotNull(getMetric("heartbeat-rate", groupName)); - assertNotNull(getMetric("heartbeat-total", groupName)); - - heartbeatMetricsManager.recordHeartbeatSentMs(time.milliseconds()); - heartbeatMetricsManager.recordRequestLatency(100); - heartbeatMetricsManager.recordRequestLatency(103); - heartbeatMetricsManager.recordRequestLatency(102); - - assertEquals(103d, getMetric("heartbeat-response-time-max", groupName).metricValue()); - assertEquals(0.1d, (double) getMetric("heartbeat-rate", groupName).metricValue(), 0.01d); - assertEquals(3d, getMetric("heartbeat-total", groupName).metricValue()); - - // Randomly sleep 1-10 seconds - Random rand = new Random(); - int randomSleepS = rand.nextInt(10) + 1; - time.sleep(TimeUnit.SECONDS.toMillis(randomSleepS)); - assertEquals((double) randomSleepS, getMetric("last-heartbeat-seconds-ago", groupName).metricValue()); - } - - @Test - public void testDefaultMetricsGroupName() { - final String coordinatorMetricsGroupName = "consumer-coordinator-metrics"; - HeartbeatMetricsManager heartbeatMetrics = new HeartbeatMetricsManager(metrics); - assertEquals(coordinatorMetricsGroupName, heartbeatMetrics.metricGroupName()); - - OffsetCommitMetricsManager commitMetrics = new OffsetCommitMetricsManager(metrics); - assertEquals(coordinatorMetricsGroupName, commitMetrics.metricGroupName()); - - RebalanceCallbackMetricsManager rebalanceCallbackMetrics = new RebalanceCallbackMetricsManager(metrics); - assertEquals(coordinatorMetricsGroupName, rebalanceCallbackMetrics.metricGroupName()); - - final String consumerMetricsGroupName = "consumer-metrics"; - KafkaConsumerMetrics consumerMetrics = new KafkaConsumerMetrics(metrics, CONSUMER_METRIC_GROUP_PREFIX); - assertEquals(consumerMetricsGroupName, consumerMetrics.metricGroupName()); - } - - private KafkaMetric getMetric(String metricName, String groupName) { - return metrics.metrics().get(metrics.metricName(metricName, groupName)); - } -} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index bcc66c1f69970..0df6fa94c39cb 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -85,7 +85,7 @@ public class HeartbeatRequestManagerTest { private int maxPollIntervalMs = DEFAULT_MAX_POLL_INTERVAL_MS; private long retryBackoffMaxMs = DEFAULT_RETRY_BACKOFF_MAX_MS; private static final String DEFAULT_GROUP_ID = "groupId"; - private String consumerMetricsGroupName = "consumer-coordinator-metrics"; + private static final String CONSUMER_COORDINATOR_METRICS = "consumer-coordinator-metrics"; private ConsumerTestBuilder testBuilder; private Time time; @@ -725,7 +725,7 @@ private ConsumerConfig config() { } private KafkaMetric getMetric(final String name) { - return metrics.metrics().get(metrics.metricName(name, consumerMetricsGroupName)); + return metrics.metrics().get(metrics.metricName(name, CONSUMER_COORDINATOR_METRICS)); } private HeartbeatRequestManager createHeartbeatRequestManager( diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManagerTest.java new file mode 100644 index 0000000000000..1761bc2fef07c --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/metrics/HeartbeatMetricsManagerTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.metrics; + +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.Time; +import org.junit.jupiter.api.Test; + +import java.util.Random; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class HeartbeatMetricsManagerTest { + private Time time = new MockTime(); + private Metrics metrics = new Metrics(time); + + @Test + public void testHeartbeatMetrics() { + // Assuming 'metrics' is an instance of your Metrics class + HeartbeatMetricsManager heartbeatMetricsManager = new HeartbeatMetricsManager(metrics); + + // Assert the existence of metrics + assertNotNull(metrics.metric(heartbeatMetricsManager.heartbeatResponseTimeMax)); + assertNotNull(metrics.metric(heartbeatMetricsManager.heartbeatRate)); + assertNotNull(metrics.metric(heartbeatMetricsManager.heartbeatTotal)); + + // Record heartbeat sent time and request latencies + long currentTimeMs = time.milliseconds(); + heartbeatMetricsManager.recordHeartbeatSentMs(currentTimeMs); + heartbeatMetricsManager.recordRequestLatency(100); + heartbeatMetricsManager.recordRequestLatency(103); + heartbeatMetricsManager.recordRequestLatency(102); + + // Assert recorded metrics values + assertEquals(103d, metrics.metric(heartbeatMetricsManager.heartbeatResponseTimeMax).metricValue()); + assertEquals(0.1d, (double) metrics.metric(heartbeatMetricsManager.heartbeatRate).metricValue(), 0.01d); + assertEquals(3d, metrics.metric(heartbeatMetricsManager.heartbeatTotal).metricValue()); + + // Randomly sleep 1-10 seconds + Random rand = new Random(); + int randomSleepS = rand.nextInt(10) + 1; + time.sleep(TimeUnit.SECONDS.toMillis(randomSleepS)); + assertEquals((double) randomSleepS, metrics.metric(heartbeatMetricsManager.lastHeartbeatSecondsAgo).metricValue()); + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManagerTest.java new file mode 100644 index 0000000000000..fa89456019a84 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/metrics/OffsetCommitMetricsManagerTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.metrics; + +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.Time; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class OffsetCommitMetricsManagerTest { + private Time time = new MockTime(); + private Metrics metrics = new Metrics(time); + + @Test + public void testOffsetCommitMetrics() { + // Assuming 'metrics' is an instance of your Metrics class + + // Create an instance of OffsetCommitMetricsManager + OffsetCommitMetricsManager metricsManager = new OffsetCommitMetricsManager(metrics); + + // Assert the existence of metrics + assertNotNull(metrics.metric(metricsManager.commitLatencyAvg)); + assertNotNull(metrics.metric(metricsManager.commitLatencyMax)); + assertNotNull(metrics.metric(metricsManager.commitRate)); + assertNotNull(metrics.metric(metricsManager.commitTotal)); + + // Record request latency + metricsManager.recordRequestLatency(100); + metricsManager.recordRequestLatency(102); + metricsManager.recordRequestLatency(98); + + // Assert the recorded latency + assertEquals(100d, metrics.metric(metricsManager.commitLatencyAvg).metricValue()); + assertEquals(102d, metrics.metric(metricsManager.commitLatencyMax).metricValue()); + assertEquals(0.1d, metrics.metric(metricsManager.commitRate).metricValue()); + assertEquals(3d, metrics.metric(metricsManager.commitTotal).metricValue()); + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManagerTest.java new file mode 100644 index 0000000000000..c0b5e6eae4824 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/metrics/RebalanceCallbackMetricsManagerTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.metrics; + +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.Time; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class RebalanceCallbackMetricsManagerTest { + private Time time = new MockTime(); + private Metrics metrics = new Metrics(time); + + @Test + public void testRebalanceCallbackMetrics() { + RebalanceCallbackMetricsManager metricsManager = new RebalanceCallbackMetricsManager(metrics); + assertNotNull(metrics.metric(metricsManager.partitionRevokeLatencyAvg)); + assertNotNull(metrics.metric(metricsManager.partitionRevokeLatencyMax)); + assertNotNull(metrics.metric(metricsManager.partitionAssignLatencyAvg)); + assertNotNull(metrics.metric(metricsManager.partitionAssignLatencyMax)); + assertNotNull(metrics.metric(metricsManager.partitionLostLatencyAvg)); + assertNotNull(metrics.metric(metricsManager.partitionLostLatencyMax)); + + metricsManager.recordPartitionsAssignedLatency(100); + metricsManager.recordPartitionsRevokedLatency(101); + metricsManager.recordPartitionsLostLatency(102); + + assertEquals(101d, metrics.metric(metricsManager.partitionRevokeLatencyAvg).metricValue()); + assertEquals(101d, metrics.metric(metricsManager.partitionRevokeLatencyMax).metricValue()); + assertEquals(100d, metrics.metric(metricsManager.partitionAssignLatencyAvg).metricValue()); + assertEquals(100d, metrics.metric(metricsManager.partitionAssignLatencyMax).metricValue()); + assertEquals(102d, metrics.metric(metricsManager.partitionLostLatencyAvg).metricValue()); + assertEquals(102d, metrics.metric(metricsManager.partitionLostLatencyMax).metricValue()); + } +}