diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java index fc9dea2ddb67c..3e52f64064af0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java @@ -338,7 +338,7 @@ default DescribeClusterResult describeCluster() { * This operation is supported by brokers with version 0.11.0.0 or higher. * * @param filter The filter to use. - * @return The DeleteAclsResult. + * @return The DescribeAclsResult. */ default DescribeAclsResult describeAcls(AclBindingFilter filter) { return describeAcls(filter, new DescribeAclsOptions()); @@ -354,7 +354,7 @@ default DescribeAclsResult describeAcls(AclBindingFilter filter) { * * @param filter The filter to use. * @param options The options to use when listing the ACLs. - * @return The DeleteAclsResult. + * @return The DescribeAclsResult. */ DescribeAclsResult describeAcls(AclBindingFilter filter, DescribeAclsOptions options); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerInterceptor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerInterceptor.java index 6af47058e4e21..c04afccd8aaf9 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerInterceptor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerInterceptor.java @@ -65,7 +65,7 @@ public interface ConsumerInterceptor extends Configurable, AutoCloseable { * @param records records to be consumed by the client or records returned by the previous interceptors in the list. * @return records that are either modified by the interceptor or same as records passed to this method. */ - public ConsumerRecords onConsume(ConsumerRecords records); + ConsumerRecords onConsume(ConsumerRecords records); /** * This is called when offsets get committed. @@ -74,10 +74,10 @@ public interface ConsumerInterceptor extends Configurable, AutoCloseable { * * @param offsets A map of offsets by partition with associated metadata */ - public void onCommit(Map offsets); + void onCommit(Map offsets); /** * This is called when interceptor is closed */ - public void close(); + void close(); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index 1f4bc7ce97751..992edc6977fc5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -1901,7 +1901,7 @@ public Map committed(final Set partitionsFor(String topic) { * @param topic The topic to get partition metadata for * @param timeout The maximum of time to await topic metadata * - * @return The list of partitions + * @return The list of partitions, which will be empty when the given topic is not found * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this * function is called * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while @@ -1948,7 +1948,7 @@ public List partitionsFor(String topic, Duration timeout) { Timer timer = time.timer(timeout); Map> topicMetadata = fetcher.getTopicMetadata( new MetadataRequest.Builder(Collections.singletonList(topic), metadata.allowAutoTopicCreation()), timer); - return topicMetadata.get(topic); + return topicMetadata.getOrDefault(topic, Collections.emptyList()); } finally { release(); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java index 345cdec00dd8e..ed29afe488c15 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java @@ -376,7 +376,7 @@ public synchronized void updateEndOffsets(final Map newOff @Override public synchronized List partitionsFor(String topic) { ensureNotClosed(); - return this.partitions.get(topic); + return this.partitions.getOrDefault(topic, Collections.emptyList()); } @Override diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java index cc013704313b5..300f5a3bd5f64 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java @@ -200,8 +200,8 @@ public class ProducerConfig extends AbstractConfig { /** max.in.flight.requests.per.connection */ public static final String MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION = "max.in.flight.requests.per.connection"; private static final String MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DOC = "The maximum number of unacknowledged requests the client will send on a single connection before blocking." - + " Note that if this setting is set to be greater than 1 and there are failed sends, there is a risk of" - + " message re-ordering due to retries (i.e., if retries are enabled)."; + + " Note that if this config is set to be greater than 1 and enable.idempotence is set to false, there is a risk of" + + " message re-ordering after a failed send due to retries (i.e., if retries are enabled)."; /** retries */ public static final String RETRIES_CONFIG = CommonClientConfigs.RETRIES_CONFIG; @@ -246,10 +246,10 @@ public class ProducerConfig extends AbstractConfig { public static final String ENABLE_IDEMPOTENCE_CONFIG = "enable.idempotence"; public static final String ENABLE_IDEMPOTENCE_DOC = "When set to 'true', the producer will ensure that exactly one copy of each message is written in the stream. If 'false', producer " + "retries due to broker failures, etc., may write duplicates of the retried message in the stream. " - + "Note that enabling idempotence requires " + MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION + " to be less than or equal to 5, " - + "" + RETRIES_CONFIG + " to be greater than 0 and " + ACKS_CONFIG + " must be 'all'. If these values " - + "are not explicitly set by the user, suitable values will be chosen. If incompatible values are set, " - + "a ConfigException will be thrown."; + + "Note that enabling idempotence requires " + MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION + " to be less than or equal to 5 " + + "(with message ordering preserved for any allowable value), " + RETRIES_CONFIG + " to be greater than 0, and " + + ACKS_CONFIG + " must be 'all'. If these values are not explicitly set by the user, suitable values will be chosen. If incompatible " + + "values are set, a ConfigException will be thrown."; /** transaction.timeout.ms */ public static final String TRANSACTION_TIMEOUT_CONFIG = "transaction.timeout.ms"; diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerInterceptor.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerInterceptor.java index f466547f4fce6..8f89d6faa9ab1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerInterceptor.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerInterceptor.java @@ -64,7 +64,7 @@ public interface ProducerInterceptor extends Configurable { * @param record the record from client or the record returned by the previous interceptor in the chain of interceptors. * @return producer record to send to topic/partition */ - public ProducerRecord onSend(ProducerRecord record); + ProducerRecord onSend(ProducerRecord record); /** * This method is called when the record sent to the server has been acknowledged, or when sending the record fails before @@ -86,10 +86,10 @@ public interface ProducerInterceptor extends Configurable { * {@link org.apache.kafka.clients.producer.KafkaProducer#send(ProducerRecord)}. * @param exception The exception thrown during processing of this record. Null if no error occurred. */ - public void onAcknowledgement(RecordMetadata metadata, Exception exception); + void onAcknowledgement(RecordMetadata metadata, Exception exception); /** * This is called when interceptor is closed */ - public void close(); + void close(); } diff --git a/clients/src/main/java/org/apache/kafka/common/compress/ZstdFactory.java b/clients/src/main/java/org/apache/kafka/common/compress/ZstdFactory.java index 8f4735e4d80da..4664f4e5656ec 100644 --- a/clients/src/main/java/org/apache/kafka/common/compress/ZstdFactory.java +++ b/clients/src/main/java/org/apache/kafka/common/compress/ZstdFactory.java @@ -17,6 +17,7 @@ package org.apache.kafka.common.compress; +import com.github.luben.zstd.BufferPool; import com.github.luben.zstd.RecyclingBufferPool; import com.github.luben.zstd.ZstdInputStreamNoFinalizer; import com.github.luben.zstd.ZstdOutputStreamNoFinalizer; @@ -47,10 +48,24 @@ public static OutputStream wrapForOutput(ByteBufferOutputStream buffer) { public static InputStream wrapForInput(ByteBuffer buffer, byte messageVersion, BufferSupplier decompressionBufferSupplier) { try { + // We use our own BufferSupplier instead of com.github.luben.zstd.RecyclingBufferPool since our + // implementation doesn't require locking or soft references. + BufferPool bufferPool = new BufferPool() { + @Override + public ByteBuffer get(int capacity) { + return decompressionBufferSupplier.get(capacity); + } + + @Override + public void release(ByteBuffer buffer) { + decompressionBufferSupplier.release(buffer); + } + }; + // Set output buffer (uncompressed) to 16 KB (none by default) to ensure reasonable performance // in cases where the caller reads a small number of bytes (potentially a single byte). return new BufferedInputStream(new ZstdInputStreamNoFinalizer(new ByteBufferInputStream(buffer), - RecyclingBufferPool.INSTANCE), 16 * 1024); + bufferPool), 16 * 1024); } catch (Throwable e) { throw new KafkaException(e); } diff --git a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java index a25c3ea521d0c..b3c7e85d5b5bc 100644 --- a/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java +++ b/clients/src/main/java/org/apache/kafka/common/config/AbstractConfig.java @@ -32,6 +32,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; /** * A convenient base class for configurations to extend. @@ -42,8 +43,12 @@ public class AbstractConfig { private final Logger log = LoggerFactory.getLogger(getClass()); - /* configs for which values have been requested, used to detect unused configs */ - private final Set used; + /** + * Configs for which values have been requested, used to detect unused configs. + * This set must be concurrent modifiable and iterable. It will be modified + * when directly accessed or as a result of RecordingMap access. + */ + private final Set used = ConcurrentHashMap.newKeySet(); /* the original values passed in by the user */ private final Map originals; @@ -106,7 +111,6 @@ public AbstractConfig(ConfigDef definition, Map originals, Map this.originals = resolveConfigVariables(configProviderProps, (Map) originals); this.values = definition.parse(this.originals); - this.used = Collections.synchronizedSet(new HashSet<>()); Map configUpdates = postProcessParsedConfig(Collections.unmodifiableMap(this.values)); for (Map.Entry update : configUpdates.entrySet()) { this.values.put(update.getKey(), update.getValue()); diff --git a/clients/src/main/java/org/apache/kafka/server/authorizer/Action.java b/clients/src/main/java/org/apache/kafka/server/authorizer/Action.java index a62b7f0965b2e..60af34b38f4bb 100644 --- a/clients/src/main/java/org/apache/kafka/server/authorizer/Action.java +++ b/clients/src/main/java/org/apache/kafka/server/authorizer/Action.java @@ -31,27 +31,32 @@ public class Action { private final boolean logIfAllowed; private final boolean logIfDenied; + /** + * @param operation non-null operation being performed + * @param resourcePattern non-null resource pattern on which this action is being performed + */ public Action(AclOperation operation, ResourcePattern resourcePattern, int resourceReferenceCount, boolean logIfAllowed, boolean logIfDenied) { - this.operation = operation; - this.resourcePattern = resourcePattern; + this.operation = Objects.requireNonNull(operation, "operation can't be null"); + this.resourcePattern = Objects.requireNonNull(resourcePattern, "resourcePattern can't be null"); this.logIfAllowed = logIfAllowed; this.logIfDenied = logIfDenied; this.resourceReferenceCount = resourceReferenceCount; } /** - * Resource on which action is being performed. + * @return a non-null resource pattern on which this action is being performed */ public ResourcePattern resourcePattern() { return resourcePattern; } /** - * Operation being performed. + * + * @return a non-null operation being performed */ public AclOperation operation() { return operation; diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 4cad255ba0241..8c987a12d02fa 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -5773,7 +5773,7 @@ public void testClientSideTimeoutAfterFailureToReceiveResponse() throws Exceptio TestUtils.waitForCondition(() -> { time.sleep(1); return disconnectFuture.isDone(); - }, 1, 5000, () -> "Timed out waiting for expected disconnect"); + }, 5000, 1, () -> "Timed out waiting for expected disconnect"); assertFalse(disconnectFuture.isCompletedExceptionally()); assertFalse(result.future.isDone()); TestUtils.waitForCondition(env.kafkaClient()::hasInFlightRequests, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index b125a91f70cad..59f72cd0eeff5 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -1757,6 +1757,28 @@ private void consumerCloseTest(final long closeTimeoutMs, } } + @Test + public void testPartitionsForNonExistingTopic() { + Time time = new MockTime(); + SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST); + ConsumerMetadata metadata = createMetadata(subscription); + MockClient client = new MockClient(time, metadata); + + initMetadata(client, Collections.singletonMap(topic, 1)); + Cluster cluster = metadata.fetch(); + + MetadataResponse updateResponse = RequestTestUtils.metadataResponse(cluster.nodes(), + cluster.clusterResource().clusterId(), + cluster.controller().id(), + Collections.emptyList()); + client.prepareResponse(updateResponse); + + ConsumerPartitionAssignor assignor = new RoundRobinAssignor(); + + KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + assertEquals(Collections.emptyList(), consumer.partitionsFor("non-exist-topic")); + } + @Test public void testPartitionsForAuthenticationFailure() { final KafkaConsumer consumer = consumerWithPendingAuthenticationError(); diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java index 5e34ae8c5b369..3c819befa5fa8 100644 --- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java +++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java @@ -298,7 +298,7 @@ public static void waitForCondition(final TestCondition testCondition, final lon * avoid transient failures due to slow or overloaded machines. */ public static void waitForCondition(final TestCondition testCondition, final long maxWaitMs, Supplier conditionDetailsSupplier) throws InterruptedException { - waitForCondition(testCondition, DEFAULT_POLL_INTERVAL_MS, maxWaitMs, conditionDetailsSupplier); + waitForCondition(testCondition, maxWaitMs, DEFAULT_POLL_INTERVAL_MS, conditionDetailsSupplier); } /** @@ -310,11 +310,11 @@ public static void waitForCondition(final TestCondition testCondition, final lon */ public static void waitForCondition( final TestCondition testCondition, - final long pollIntervalMs, final long maxWaitMs, + final long pollIntervalMs, Supplier conditionDetailsSupplier ) throws InterruptedException { - retryOnExceptionWithTimeout(pollIntervalMs, maxWaitMs, () -> { + retryOnExceptionWithTimeout(maxWaitMs, pollIntervalMs, () -> { String conditionDetailsSupplied = conditionDetailsSupplier != null ? conditionDetailsSupplier.get() : null; String conditionDetails = conditionDetailsSupplied != null ? conditionDetailsSupplied : ""; assertTrue(testCondition.conditionMet(), @@ -333,7 +333,7 @@ public static void waitForCondition( */ public static void retryOnExceptionWithTimeout(final long timeoutMs, final ValuelessCallable runnable) throws InterruptedException { - retryOnExceptionWithTimeout(DEFAULT_POLL_INTERVAL_MS, timeoutMs, runnable); + retryOnExceptionWithTimeout(timeoutMs, DEFAULT_POLL_INTERVAL_MS, runnable); } /** @@ -345,7 +345,7 @@ public static void retryOnExceptionWithTimeout(final long timeoutMs, * @throws InterruptedException if the current thread is interrupted while waiting for {@code runnable} to complete successfully. */ public static void retryOnExceptionWithTimeout(final ValuelessCallable runnable) throws InterruptedException { - retryOnExceptionWithTimeout(DEFAULT_POLL_INTERVAL_MS, DEFAULT_MAX_WAIT_MS, runnable); + retryOnExceptionWithTimeout(DEFAULT_MAX_WAIT_MS, DEFAULT_POLL_INTERVAL_MS, runnable); } /** @@ -353,13 +353,13 @@ public static void retryOnExceptionWithTimeout(final ValuelessCallable runnable) * {@link AssertionError}s, or for the given timeout to expire. If the timeout expires then the * last exception or assertion failure will be thrown thus providing context for the failure. * - * @param pollIntervalMs the interval in milliseconds to wait between invoking {@code runnable}. * @param timeoutMs the total time in milliseconds to wait for {@code runnable} to complete successfully. + * @param pollIntervalMs the interval in milliseconds to wait between invoking {@code runnable}. * @param runnable the code to attempt to execute successfully. * @throws InterruptedException if the current thread is interrupted while waiting for {@code runnable} to complete successfully. */ - public static void retryOnExceptionWithTimeout(final long pollIntervalMs, - final long timeoutMs, + public static void retryOnExceptionWithTimeout(final long timeoutMs, + final long pollIntervalMs, final ValuelessCallable runnable) throws InterruptedException { final long expectedEnd = System.currentTimeMillis() + timeoutMs; diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/CheckpointTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/CheckpointTest.java index fd5448c13f93a..f008f996772ce 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/CheckpointTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/CheckpointTest.java @@ -32,9 +32,13 @@ public void testSerde() { byte[] value = checkpoint.recordValue(); ConsumerRecord record = new ConsumerRecord<>("any-topic", 7, 8, key, value); Checkpoint deserialized = Checkpoint.deserializeRecord(record); - assertEquals(checkpoint.consumerGroupId(), deserialized.consumerGroupId()); - assertEquals(checkpoint.topicPartition(), deserialized.topicPartition()); - assertEquals(checkpoint.upstreamOffset(), deserialized.upstreamOffset()); - assertEquals(checkpoint.downstreamOffset(), deserialized.downstreamOffset()); + assertEquals(checkpoint.consumerGroupId(), deserialized.consumerGroupId(), + "Failure on checkpoint consumerGroupId serde"); + assertEquals(checkpoint.topicPartition(), deserialized.topicPartition(), + "Failure on checkpoint topicPartition serde"); + assertEquals(checkpoint.upstreamOffset(), deserialized.upstreamOffset(), + "Failure on checkpoint upstreamOffset serde"); + assertEquals(checkpoint.downstreamOffset(), deserialized.downstreamOffset(), + "Failure on checkpoint downstreamOffset serde"); } } diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/HeartbeatTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/HeartbeatTest.java index fb473f7b0af0c..723b0dc2bfe56 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/HeartbeatTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/HeartbeatTest.java @@ -31,8 +31,11 @@ public void testSerde() { byte[] value = heartbeat.recordValue(); ConsumerRecord record = new ConsumerRecord<>("any-topic", 6, 7, key, value); Heartbeat deserialized = Heartbeat.deserializeRecord(record); - assertEquals(heartbeat.sourceClusterAlias(), deserialized.sourceClusterAlias()); - assertEquals(heartbeat.targetClusterAlias(), deserialized.targetClusterAlias()); - assertEquals(heartbeat.timestamp(), deserialized.timestamp()); + assertEquals(heartbeat.sourceClusterAlias(), deserialized.sourceClusterAlias(), + "Failure on heartbeat sourceClusterAlias serde"); + assertEquals(heartbeat.targetClusterAlias(), deserialized.targetClusterAlias(), + "Failure on heartbeat targetClusterAlias serde"); + assertEquals(heartbeat.timestamp(), deserialized.timestamp(), + "Failure on heartbeat timestamp serde"); } } diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorCheckpointConnectorTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorCheckpointConnectorTest.java index 3c8453c77518a..1391e7615d3a0 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorCheckpointConnectorTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorCheckpointConnectorTest.java @@ -38,6 +38,8 @@ public class MirrorCheckpointConnectorTest { + private static final String CONSUMER_GROUP = "consumer-group-1"; + @Test public void testMirrorCheckpointConnectorDisabled() { // disable the checkpoint emission @@ -45,13 +47,13 @@ public void testMirrorCheckpointConnectorDisabled() { makeProps("emit.checkpoints.enabled", "false")); List knownConsumerGroups = new ArrayList<>(); - knownConsumerGroups.add("consumer-group-1"); + knownConsumerGroups.add(CONSUMER_GROUP); // MirrorCheckpointConnector as minimum to run taskConfig() MirrorCheckpointConnector connector = new MirrorCheckpointConnector(knownConsumerGroups, config); List> output = connector.taskConfigs(1); // expect no task will be created - assertEquals(0, output.size()); + assertEquals(0, output.size(), "MirrorCheckpointConnector not disabled"); } @Test @@ -61,14 +63,16 @@ public void testMirrorCheckpointConnectorEnabled() { makeProps("emit.checkpoints.enabled", "true")); List knownConsumerGroups = new ArrayList<>(); - knownConsumerGroups.add("consumer-group-1"); + knownConsumerGroups.add(CONSUMER_GROUP); // MirrorCheckpointConnector as minimum to run taskConfig() MirrorCheckpointConnector connector = new MirrorCheckpointConnector(knownConsumerGroups, config); List> output = connector.taskConfigs(1); // expect 1 task will be created - assertEquals(1, output.size()); - assertEquals("consumer-group-1", output.get(0).get(MirrorConnectorConfig.TASK_CONSUMER_GROUPS)); + assertEquals(1, output.size(), + "MirrorCheckpointConnectorEnabled for " + CONSUMER_GROUP + " has incorrect size"); + assertEquals(CONSUMER_GROUP, output.get(0).get(MirrorConnectorConfig.TASK_CONSUMER_GROUPS), + "MirrorCheckpointConnectorEnabled for " + CONSUMER_GROUP + " failed"); } @Test @@ -77,7 +81,7 @@ public void testNoConsumerGroup() { MirrorCheckpointConnector connector = new MirrorCheckpointConnector(new ArrayList<>(), config); List> output = connector.taskConfigs(1); // expect no task will be created - assertEquals(0, output.size()); + assertEquals(0, output.size(), "ConsumerGroup shouldn't exist"); } @Test @@ -86,12 +90,12 @@ public void testReplicationDisabled() { MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps("enabled", "false")); List knownConsumerGroups = new ArrayList<>(); - knownConsumerGroups.add("consumer-group-1"); + knownConsumerGroups.add(CONSUMER_GROUP); // MirrorCheckpointConnector as minimum to run taskConfig() MirrorCheckpointConnector connector = new MirrorCheckpointConnector(knownConsumerGroups, config); List> output = connector.taskConfigs(1); // expect no task will be created - assertEquals(0, output.size()); + assertEquals(0, output.size(), "Replication isn't disabled"); } @Test @@ -100,13 +104,14 @@ public void testReplicationEnabled() { MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps("enabled", "true")); List knownConsumerGroups = new ArrayList<>(); - knownConsumerGroups.add("consumer-group-1"); + knownConsumerGroups.add(CONSUMER_GROUP); // MirrorCheckpointConnector as minimum to run taskConfig() MirrorCheckpointConnector connector = new MirrorCheckpointConnector(knownConsumerGroups, config); List> output = connector.taskConfigs(1); // expect 1 task will be created - assertEquals(1, output.size()); - assertEquals("consumer-group-1", output.get(0).get(MirrorConnectorConfig.TASK_CONSUMER_GROUPS)); + assertEquals(1, output.size(), "Replication for consumer-group-1 has incorrect size"); + assertEquals(CONSUMER_GROUP, output.get(0).get(MirrorConnectorConfig.TASK_CONSUMER_GROUPS), + "Replication for consumer-group-1 failed"); } @Test @@ -123,7 +128,8 @@ public void testFindConsumerGroups() throws Exception { List groupFound = connector.findConsumerGroups(); Set expectedGroups = groups.stream().map(ConsumerGroupListing::groupId).collect(Collectors.toSet()); - assertEquals(expectedGroups, new HashSet<>(groupFound)); + assertEquals(expectedGroups, new HashSet<>(groupFound), + "Expected groups are not the same as findConsumerGroups"); } } diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorCheckpointTaskTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorCheckpointTaskTest.java index abd314bb71c9e..7ef878ab2e8d3 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorCheckpointTaskTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorCheckpointTaskTest.java @@ -36,11 +36,14 @@ public void testDownstreamTopicRenaming() { MirrorCheckpointTask mirrorCheckpointTask = new MirrorCheckpointTask("source1", "target2", new DefaultReplicationPolicy(), null, Collections.emptyMap(), Collections.emptyMap()); assertEquals(new TopicPartition("source1.topic3", 4), - mirrorCheckpointTask.renameTopicPartition(new TopicPartition("topic3", 4))); + mirrorCheckpointTask.renameTopicPartition(new TopicPartition("topic3", 4)), + "Renaming source1.topic3 failed"); assertEquals(new TopicPartition("topic3", 5), - mirrorCheckpointTask.renameTopicPartition(new TopicPartition("target2.topic3", 5))); + mirrorCheckpointTask.renameTopicPartition(new TopicPartition("target2.topic3", 5)), + "Renaming target2.topic3 failed"); assertEquals(new TopicPartition("source1.source6.topic7", 8), - mirrorCheckpointTask.renameTopicPartition(new TopicPartition("source6.topic7", 8))); + mirrorCheckpointTask.renameTopicPartition(new TopicPartition("source6.topic7", 8)), + "Renaming source1.source6.topic7 failed"); } @Test @@ -53,21 +56,33 @@ public void testCheckpoint() { Checkpoint checkpoint1 = mirrorCheckpointTask.checkpoint("group9", new TopicPartition("topic1", 2), new OffsetAndMetadata(10, null)); SourceRecord sourceRecord1 = mirrorCheckpointTask.checkpointRecord(checkpoint1, 123L); - assertEquals(new TopicPartition("source1.topic1", 2), checkpoint1.topicPartition()); - assertEquals("group9", checkpoint1.consumerGroupId()); - assertEquals("group9", Checkpoint.unwrapGroup(sourceRecord1.sourcePartition())); - assertEquals(10, checkpoint1.upstreamOffset()); - assertEquals(11, checkpoint1.downstreamOffset()); - assertEquals(123L, sourceRecord1.timestamp().longValue()); + assertEquals(new TopicPartition("source1.topic1", 2), checkpoint1.topicPartition(), + "checkpoint group9 source1.topic1 failed"); + assertEquals("group9", checkpoint1.consumerGroupId(), + "checkpoint group9 consumerGroupId failed"); + assertEquals("group9", Checkpoint.unwrapGroup(sourceRecord1.sourcePartition()), + "checkpoint group9 sourcePartition failed"); + assertEquals(10, checkpoint1.upstreamOffset(), + "checkpoint group9 upstreamOffset failed"); + assertEquals(11, checkpoint1.downstreamOffset(), + "checkpoint group9 downstreamOffset failed"); + assertEquals(123L, sourceRecord1.timestamp().longValue(), + "checkpoint group9 timestamp failed"); Checkpoint checkpoint2 = mirrorCheckpointTask.checkpoint("group11", new TopicPartition("target2.topic5", 6), new OffsetAndMetadata(12, null)); SourceRecord sourceRecord2 = mirrorCheckpointTask.checkpointRecord(checkpoint2, 234L); - assertEquals(new TopicPartition("topic5", 6), checkpoint2.topicPartition()); - assertEquals("group11", checkpoint2.consumerGroupId()); - assertEquals("group11", Checkpoint.unwrapGroup(sourceRecord2.sourcePartition())); - assertEquals(12, checkpoint2.upstreamOffset()); - assertEquals(13, checkpoint2.downstreamOffset()); - assertEquals(234L, sourceRecord2.timestamp().longValue()); + assertEquals(new TopicPartition("topic5", 6), checkpoint2.topicPartition(), + "checkpoint group11 topic5 failed"); + assertEquals("group11", checkpoint2.consumerGroupId(), + "checkpoint group11 consumerGroupId failed"); + assertEquals("group11", Checkpoint.unwrapGroup(sourceRecord2.sourcePartition()), + "checkpoint group11 sourcePartition failed"); + assertEquals(12, checkpoint2.upstreamOffset(), + "checkpoint group11 upstreamOffset failed"); + assertEquals(13, checkpoint2.downstreamOffset(), + "checkpoint group11 downstreamOffset failed"); + assertEquals(234L, sourceRecord2.timestamp().longValue(), + "checkpoint group11 timestamp failed"); } @Test @@ -118,7 +133,9 @@ public void testSyncOffset() { Map> output = mirrorCheckpointTask.syncGroupOffset(); - assertEquals(101, output.get(consumer1).get(t1p0).offset()); - assertEquals(51, output.get(consumer2).get(t2p0).offset()); + assertEquals(101, output.get(consumer1).get(t1p0).offset(), + "Consumer 1 " + topic1 + " failed"); + assertEquals(51, output.get(consumer2).get(t2p0).offset(), + "Consumer 2 " + topic2 + " failed"); } } diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorConfigTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorConfigTest.java index f53aa68387276..7abe30def6ddc 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorConfigTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorConfigTest.java @@ -41,7 +41,8 @@ public void testTaskConfigTopicPartitions() { MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps()); Map props = config.taskConfigForTopicPartitions(topicPartitions); MirrorTaskConfig taskConfig = new MirrorTaskConfig(props); - assertEquals(taskConfig.taskTopicPartitions(), new HashSet<>(topicPartitions)); + assertEquals(taskConfig.taskTopicPartitions(), new HashSet<>(topicPartitions), + "Setting topic property configuration failed"); } @Test @@ -50,29 +51,36 @@ public void testTaskConfigConsumerGroups() { MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps()); Map props = config.taskConfigForConsumerGroups(groups); MirrorTaskConfig taskConfig = new MirrorTaskConfig(props); - assertEquals(taskConfig.taskConsumerGroups(), new HashSet<>(groups)); + assertEquals(taskConfig.taskConsumerGroups(), new HashSet<>(groups), + "Setting consumer groups property configuration failed"); } @Test public void testTopicMatching() { MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps("topics", "topic1")); - assertTrue(config.topicFilter().shouldReplicateTopic("topic1")); - assertFalse(config.topicFilter().shouldReplicateTopic("topic2")); + assertTrue(config.topicFilter().shouldReplicateTopic("topic1"), + "topic1 replication property configuration failed"); + assertFalse(config.topicFilter().shouldReplicateTopic("topic2"), + "topic2 replication property configuration failed"); } @Test public void testGroupMatching() { MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps("groups", "group1")); - assertTrue(config.groupFilter().shouldReplicateGroup("group1")); - assertFalse(config.groupFilter().shouldReplicateGroup("group2")); + assertTrue(config.groupFilter().shouldReplicateGroup("group1"), + "topic1 group matching property configuration failed"); + assertFalse(config.groupFilter().shouldReplicateGroup("group2"), + "topic2 group matching property configuration failed"); } @Test public void testConfigPropertyMatching() { MirrorConnectorConfig config = new MirrorConnectorConfig( makeProps("config.properties.exclude", "prop2")); - assertTrue(config.configPropertyFilter().shouldReplicateConfigProperty("prop1")); - assertFalse(config.configPropertyFilter().shouldReplicateConfigProperty("prop2")); + assertTrue(config.configPropertyFilter().shouldReplicateConfigProperty("prop1"), + "config.properties.exclude incorrectly excluded prop1"); + assertFalse(config.configPropertyFilter().shouldReplicateConfigProperty("prop2"), + "config.properties.exclude incorrectly included prop2"); } @Test @@ -92,24 +100,26 @@ public void testConfigBackwardsCompatibility() { @Test public void testNoTopics() { MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps("topics", "")); - assertFalse(config.topicFilter().shouldReplicateTopic("topic1")); - assertFalse(config.topicFilter().shouldReplicateTopic("topic2")); - assertFalse(config.topicFilter().shouldReplicateTopic("")); + assertFalse(config.topicFilter().shouldReplicateTopic("topic1"), "topic1 shouldn't exist"); + assertFalse(config.topicFilter().shouldReplicateTopic("topic2"), "topic2 shouldn't exist"); + assertFalse(config.topicFilter().shouldReplicateTopic(""), "Empty topic shouldn't exist"); } @Test public void testAllTopics() { MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps("topics", ".*")); - assertTrue(config.topicFilter().shouldReplicateTopic("topic1")); - assertTrue(config.topicFilter().shouldReplicateTopic("topic2")); + assertTrue(config.topicFilter().shouldReplicateTopic("topic1"), + "topic1 created from wildcard should exist"); + assertTrue(config.topicFilter().shouldReplicateTopic("topic2"), + "topic2 created from wildcard should exist"); } @Test public void testListOfTopics() { MirrorConnectorConfig config = new MirrorConnectorConfig(makeProps("topics", "topic1, topic2")); - assertTrue(config.topicFilter().shouldReplicateTopic("topic1")); - assertTrue(config.topicFilter().shouldReplicateTopic("topic2")); - assertFalse(config.topicFilter().shouldReplicateTopic("topic3")); + assertTrue(config.topicFilter().shouldReplicateTopic("topic1"), "topic1 created from list should exist"); + assertTrue(config.topicFilter().shouldReplicateTopic("topic2"), "topic2 created from list should exist"); + assertFalse(config.topicFilter().shouldReplicateTopic("topic3"), "topic3 created from list should exist"); } @Test @@ -156,7 +166,8 @@ public void testSourceConsumerConfig() { connectorConsumerProps = config.sourceConsumerConfig(); expectedConsumerProps.put("auto.offset.reset", "latest"); expectedConsumerProps.remove("max.poll.interval.ms"); - assertEquals(expectedConsumerProps, connectorConsumerProps); + assertEquals(expectedConsumerProps, connectorConsumerProps, + MirrorConnectorConfig.CONSUMER_CLIENT_PREFIX + " source consumer config not matching"); } @Test @@ -172,7 +183,8 @@ public void testSourceConsumerConfigWithSourcePrefix() { expectedConsumerProps.put("enable.auto.commit", "false"); expectedConsumerProps.put("auto.offset.reset", "latest"); expectedConsumerProps.put("max.poll.interval.ms", "100"); - assertEquals(expectedConsumerProps, connectorConsumerProps); + assertEquals(expectedConsumerProps, connectorConsumerProps, + prefix + " source consumer config not matching"); } @Test @@ -184,7 +196,8 @@ public void testSourceProducerConfig() { Map connectorProducerProps = config.sourceProducerConfig(); Map expectedProducerProps = new HashMap<>(); expectedProducerProps.put("acks", "1"); - assertEquals(expectedProducerProps, connectorProducerProps); + assertEquals(expectedProducerProps, connectorProducerProps, + MirrorConnectorConfig.PRODUCER_CLIENT_PREFIX + " source product config not matching"); } @Test @@ -195,7 +208,8 @@ public void testSourceProducerConfigWithSourcePrefix() { Map connectorProducerProps = config.sourceProducerConfig(); Map expectedProducerProps = new HashMap<>(); expectedProducerProps.put("acks", "1"); - assertEquals(expectedProducerProps, connectorProducerProps); + assertEquals(expectedProducerProps, connectorProducerProps, + prefix + " source producer config not matching"); } @Test @@ -208,7 +222,8 @@ public void testSourceAdminConfig() { Map connectorAdminProps = config.sourceAdminConfig(); Map expectedAdminProps = new HashMap<>(); expectedAdminProps.put("connections.max.idle.ms", "10000"); - assertEquals(expectedAdminProps, connectorAdminProps); + assertEquals(expectedAdminProps, connectorAdminProps, + MirrorConnectorConfig.ADMIN_CLIENT_PREFIX + " source connector admin props not matching"); } @Test @@ -219,7 +234,7 @@ public void testSourceAdminConfigWithSourcePrefix() { Map connectorAdminProps = config.sourceAdminConfig(); Map expectedAdminProps = new HashMap<>(); expectedAdminProps.put("connections.max.idle.ms", "10000"); - assertEquals(expectedAdminProps, connectorAdminProps); + assertEquals(expectedAdminProps, connectorAdminProps, prefix + " source connector admin props not matching"); } @Test @@ -232,7 +247,8 @@ public void testTargetAdminConfig() { Map connectorAdminProps = config.targetAdminConfig(); Map expectedAdminProps = new HashMap<>(); expectedAdminProps.put("connections.max.idle.ms", "10000"); - assertEquals(expectedAdminProps, connectorAdminProps); + assertEquals(expectedAdminProps, connectorAdminProps, + MirrorConnectorConfig.ADMIN_CLIENT_PREFIX + " target connector admin props not matching"); } @Test @@ -243,7 +259,7 @@ public void testTargetAdminConfigWithSourcePrefix() { Map connectorAdminProps = config.targetAdminConfig(); Map expectedAdminProps = new HashMap<>(); expectedAdminProps.put("connections.max.idle.ms", "10000"); - assertEquals(expectedAdminProps, connectorAdminProps); + assertEquals(expectedAdminProps, connectorAdminProps, prefix + " source connector admin props not matching"); } } diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorHeartBeatConnectorTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorHeartBeatConnectorTest.java index b48c46977078e..ec0691983a63f 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorHeartBeatConnectorTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorHeartBeatConnectorTest.java @@ -34,7 +34,7 @@ public void testMirrorHeartbeatConnectorDisabled() { MirrorHeartbeatConnector connector = new MirrorHeartbeatConnector(config); List> output = connector.taskConfigs(1); // expect no task will be created - assertEquals(0, output.size()); + assertEquals(0, output.size(), "Expected task to not be created"); } @Test @@ -47,6 +47,6 @@ public void testReplicationDisabled() { MirrorHeartbeatConnector connector = new MirrorHeartbeatConnector(config); List> output = connector.taskConfigs(1); // expect one task will be created, even the replication is disabled - assertEquals(1, output.size()); + assertEquals(1, output.size(), "Task should have been created even with replication disabled"); } } diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorHeartbeatTaskTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorHeartbeatTaskTest.java index d4f96e785b9f4..39fd6dff10e30 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorHeartbeatTaskTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorHeartbeatTaskTest.java @@ -34,7 +34,9 @@ public void testPollCreatesRecords() throws InterruptedException { List records = heartbeatTask.poll(); assertEquals(1, records.size()); Map sourcePartition = records.iterator().next().sourcePartition(); - assertEquals(sourcePartition.get(Heartbeat.SOURCE_CLUSTER_ALIAS_KEY), "testSource"); - assertEquals(sourcePartition.get(Heartbeat.TARGET_CLUSTER_ALIAS_KEY), "testTarget"); + assertEquals(sourcePartition.get(Heartbeat.SOURCE_CLUSTER_ALIAS_KEY), "testSource", + "sourcePartition's " + Heartbeat.SOURCE_CLUSTER_ALIAS_KEY + " record was not created"); + assertEquals(sourcePartition.get(Heartbeat.TARGET_CLUSTER_ALIAS_KEY), "testTarget", + "sourcePartition's " + Heartbeat.TARGET_CLUSTER_ALIAS_KEY + " record was not created"); } -} \ No newline at end of file +} diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java index f5fe2c3ca32ec..4787ecd95ebda 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java @@ -137,13 +137,13 @@ public void testIncludesConnectorConfigProperties() { MirrorConnectorConfig connectorConfig = new MirrorConnectorConfig(connectorProps); assertEquals(100, (int) connectorConfig.getInt("tasks.max"), "Connector properties like tasks.max should be passed through to underlying Connectors."); - assertEquals(Arrays.asList("topic-1"), connectorConfig.getList("topics"), + assertEquals(Collections.singletonList("topic-1"), connectorConfig.getList("topics"), "Topics include should be passed through to underlying Connectors."); - assertEquals(Arrays.asList("group-2"), connectorConfig.getList("groups"), + assertEquals(Collections.singletonList("group-2"), connectorConfig.getList("groups"), "Groups include should be passed through to underlying Connectors."); - assertEquals(Arrays.asList("property-3"), connectorConfig.getList("config.properties.exclude"), + assertEquals(Collections.singletonList("property-3"), connectorConfig.getList("config.properties.exclude"), "Config properties exclude should be passed through to underlying Connectors."); - assertEquals(Arrays.asList("FakeMetricsReporter"), connectorConfig.getList("metric.reporters"), + assertEquals(Collections.singletonList("FakeMetricsReporter"), connectorConfig.getList("metric.reporters"), "Metrics reporters should be passed through to underlying Connectors."); assertEquals("DefaultTopicFilter", connectorConfig.getClass("topic.filter.class").getSimpleName(), "Filters should be passed through to underlying Connectors."); @@ -168,13 +168,13 @@ public void testConfigBackwardsCompatibility() { DefaultTopicFilter.TopicFilterConfig filterConfig = new DefaultTopicFilter.TopicFilterConfig(connectorProps); - assertEquals(Arrays.asList("topic3"), filterConfig.getList("topics.exclude"), + assertEquals(Collections.singletonList("topic3"), filterConfig.getList("topics.exclude"), "Topics exclude should be backwards compatible."); - assertEquals(Arrays.asList("group-7"), connectorConfig.getList("groups.exclude"), + assertEquals(Collections.singletonList("group-7"), connectorConfig.getList("groups.exclude"), "Groups exclude should be backwards compatible."); - assertEquals(Arrays.asList("property-3"), connectorConfig.getList("config.properties.exclude"), + assertEquals(Collections.singletonList("property-3"), connectorConfig.getList("config.properties.exclude"), "Config properties exclude should be backwards compatible."); } @@ -193,10 +193,10 @@ public void testConfigBackwardsCompatibilitySourceTarget() { DefaultTopicFilter.TopicFilterConfig filterConfig = new DefaultTopicFilter.TopicFilterConfig(connectorProps); - assertEquals(Arrays.asList("topic3"), filterConfig.getList("topics.exclude"), + assertEquals(Collections.singletonList("topic3"), filterConfig.getList("topics.exclude"), "Topics exclude should be backwards compatible."); - assertEquals(Arrays.asList("group-7"), connectorConfig.getList("groups.exclude"), + assertEquals(Collections.singletonList("group-7"), connectorConfig.getList("groups.exclude"), "Groups exclude should be backwards compatible."); } @@ -213,7 +213,7 @@ public void testIncludesTopicFilterProperties() { new DefaultTopicFilter.TopicFilterConfig(connectorProps); assertEquals(Arrays.asList("topic1", "topic2"), filterConfig.getList("topics"), "source->target.topics should be passed through to TopicFilters."); - assertEquals(Arrays.asList("topic3"), filterConfig.getList("topics.exclude"), + assertEquals(Collections.singletonList("topic3"), filterConfig.getList("topics.exclude"), "source->target.topics.exclude should be passed through to TopicFilters."); } diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java index 42d7951cd60fc..68d149c755ff8 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java @@ -167,13 +167,13 @@ public void testMirrorSourceConnectorTaskConfig() { // t3 -> [t0p2, t0p5, t1p0, t2p1] Map t1 = output.get(0); - assertEquals("t0-0,t0-3,t0-6,t1-1", t1.get(TASK_TOPIC_PARTITIONS)); + assertEquals("t0-0,t0-3,t0-6,t1-1", t1.get(TASK_TOPIC_PARTITIONS), "Config for t1 is incorrect"); Map t2 = output.get(1); - assertEquals("t0-1,t0-4,t0-7,t2-0", t2.get(TASK_TOPIC_PARTITIONS)); + assertEquals("t0-1,t0-4,t0-7,t2-0", t2.get(TASK_TOPIC_PARTITIONS), "Config for t2 is incorrect"); Map t3 = output.get(2); - assertEquals("t0-2,t0-5,t1-0,t2-1", t3.get(TASK_TOPIC_PARTITIONS)); + assertEquals("t0-2,t0-5,t1-0,t2-1", t3.get(TASK_TOPIC_PARTITIONS), "Config for t3 is incorrect"); } @Test @@ -201,7 +201,7 @@ public void testRefreshTopicPartitions() throws Exception { Map expectedPartitionCounts = new HashMap<>(); expectedPartitionCounts.put("source.topic", 1L); Map configMap = MirrorSourceConnector.configToMap(topicConfig); - assertEquals(2, configMap.size()); + assertEquals(2, configMap.size(), "configMap has incorrect size"); Map expectedNewTopics = new HashMap<>(); expectedNewTopics.put("source.topic", new NewTopic("source.topic", 1, (short) 0).configs(configMap)); diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceTaskTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceTaskTest.java index 9cf09f8c4cfd0..feb2f7fb6ba68 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceTaskTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceTaskTest.java @@ -54,15 +54,22 @@ public void testSerde() { MirrorSourceTask mirrorSourceTask = new MirrorSourceTask(null, null, "cluster7", new DefaultReplicationPolicy(), 50); SourceRecord sourceRecord = mirrorSourceTask.convertRecord(consumerRecord); - assertEquals("cluster7.topic1", sourceRecord.topic()); - assertEquals(2, sourceRecord.kafkaPartition().intValue()); - assertEquals(new TopicPartition("topic1", 2), MirrorUtils.unwrapPartition(sourceRecord.sourcePartition())); - assertEquals(3L, MirrorUtils.unwrapOffset(sourceRecord.sourceOffset()).longValue()); - assertEquals(4L, sourceRecord.timestamp().longValue()); - assertEquals(key, sourceRecord.key()); - assertEquals(value, sourceRecord.value()); - assertEquals(headers.lastHeader("header1").value(), sourceRecord.headers().lastWithName("header1").value()); - assertEquals(headers.lastHeader("header2").value(), sourceRecord.headers().lastWithName("header2").value()); + assertEquals("cluster7.topic1", sourceRecord.topic(), + "Failure on cluster7.topic1 consumerRecord serde"); + assertEquals(2, sourceRecord.kafkaPartition().intValue(), + "sourceRecord kafka partition is incorrect"); + assertEquals(new TopicPartition("topic1", 2), MirrorUtils.unwrapPartition(sourceRecord.sourcePartition()), + "topic1 unwrapped from sourcePartition is incorrect"); + assertEquals(3L, MirrorUtils.unwrapOffset(sourceRecord.sourceOffset()).longValue(), + "sourceRecord's sourceOffset is incorrect"); + assertEquals(4L, sourceRecord.timestamp().longValue(), + "sourceRecord's timestamp is incorrect"); + assertEquals(key, sourceRecord.key(), "sourceRecord's key is incorrect"); + assertEquals(value, sourceRecord.value(), "sourceRecord's value is incorrect"); + assertEquals(headers.lastHeader("header1").value(), sourceRecord.headers().lastWithName("header1").value(), + "sourceRecord's header1 is incorrect"); + assertEquals(headers.lastHeader("header2").value(), sourceRecord.headers().lastWithName("header2").value(), + "sourceRecord's header2 is incorrect"); } @Test @@ -86,16 +93,16 @@ public void testZeroOffsetSync() { MirrorSourceTask.PartitionState partitionState = new MirrorSourceTask.PartitionState(0); // if max offset lag is zero, should always emit offset syncs - assertTrue(partitionState.update(0, 100)); - assertTrue(partitionState.update(2, 102)); - assertTrue(partitionState.update(3, 153)); - assertTrue(partitionState.update(4, 154)); - assertTrue(partitionState.update(5, 155)); - assertTrue(partitionState.update(6, 207)); - assertTrue(partitionState.update(2, 208)); - assertTrue(partitionState.update(3, 209)); - assertTrue(partitionState.update(4, 3)); - assertTrue(partitionState.update(5, 4)); + assertTrue(partitionState.update(0, 100), "zeroOffsetSync downStreamOffset 100 is incorrect"); + assertTrue(partitionState.update(2, 102), "zeroOffsetSync downStreamOffset 102 is incorrect"); + assertTrue(partitionState.update(3, 153), "zeroOffsetSync downStreamOffset 153 is incorrect"); + assertTrue(partitionState.update(4, 154), "zeroOffsetSync downStreamOffset 154 is incorrect"); + assertTrue(partitionState.update(5, 155), "zeroOffsetSync downStreamOffset 155 is incorrect"); + assertTrue(partitionState.update(6, 207), "zeroOffsetSync downStreamOffset 207 is incorrect"); + assertTrue(partitionState.update(2, 208), "zeroOffsetSync downStreamOffset 208 is incorrect"); + assertTrue(partitionState.update(3, 209), "zeroOffsetSync downStreamOffset 209 is incorrect"); + assertTrue(partitionState.update(4, 3), "zeroOffsetSync downStreamOffset 3 is incorrect"); + assertTrue(partitionState.update(5, 4), "zeroOffsetSync downStreamOffset 4 is incorrect"); } @Test @@ -134,13 +141,16 @@ public void testPoll() { for (int i = 0; i < sourceRecords.size(); i++) { SourceRecord sourceRecord = sourceRecords.get(i); ConsumerRecord consumerRecord = consumerRecordsList.get(i); - assertEquals(consumerRecord.key(), sourceRecord.key()); - assertEquals(consumerRecord.value(), sourceRecord.value()); + assertEquals(consumerRecord.key(), sourceRecord.key(), + "consumerRecord key does not equal sourceRecord key"); + assertEquals(consumerRecord.value(), sourceRecord.value(), + "consumerRecord value does not equal sourceRecord value"); // We expect that the topicname will be based on the replication policy currently used assertEquals(replicationPolicy.formatRemoteTopic(sourceClusterName, topicName), - sourceRecord.topic()); + sourceRecord.topic(), "topicName not the same as the current replicationPolicy"); // We expect that MirrorMaker will keep the same partition assignment - assertEquals(consumerRecord.partition(), sourceRecord.kafkaPartition().intValue()); + assertEquals(consumerRecord.partition(), sourceRecord.kafkaPartition().intValue(), + "partition assignment not the same as the current replicationPolicy"); // Check header values List
expectedHeaders = new ArrayList<>(); consumerRecord.headers().forEach(expectedHeaders::add); @@ -155,8 +165,10 @@ private void compareHeaders(List
expectedHeaders, List can't translate - assertEquals(-1, store.translateDownstream(tp, 5)); + assertEquals(-1, store.translateDownstream(tp, 5), + "Expected old offset to not translate"); // Downstream offsets reset store.sync(tp, 200, 10); - assertEquals(store.translateDownstream(tp, 200), 10); + assertEquals(store.translateDownstream(tp, 200), 10, + "Failure in resetting translation of downstream offset"); // Upstream offsets reset store.sync(tp, 20, 20); - assertEquals(store.translateDownstream(tp, 20), 20); + assertEquals(store.translateDownstream(tp, 20), 20, + "Failure in resetting translation of upstream offset"); } } diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncTest.java index 33f3ab0d2be51..dc7efe291af83 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncTest.java @@ -32,8 +32,11 @@ public void testSerde() { byte[] value = offsetSync.recordValue(); ConsumerRecord record = new ConsumerRecord<>("any-topic", 6, 7, key, value); OffsetSync deserialized = OffsetSync.deserializeRecord(record); - assertEquals(offsetSync.topicPartition(), deserialized.topicPartition()); - assertEquals(offsetSync.upstreamOffset(), deserialized.upstreamOffset()); - assertEquals(offsetSync.downstreamOffset(), deserialized.downstreamOffset()); + assertEquals(offsetSync.topicPartition(), deserialized.topicPartition(), + "Failure on offset sync topic partition serde"); + assertEquals(offsetSync.upstreamOffset(), deserialized.upstreamOffset(), + "Failure on upstream offset serde"); + assertEquals(offsetSync.downstreamOffset(), deserialized.downstreamOffset(), + "Failure on downstream offset serde"); } } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java index 6e2350fae0894..b1920d59b20e6 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java @@ -175,12 +175,12 @@ public void start() { List partitionInfos = consumer.partitionsFor(topic); long started = time.nanoseconds(); long sleepMs = 100; - while (partitionInfos == null && time.nanoseconds() - started < CREATE_TOPIC_TIMEOUT_NS) { + while (partitionInfos.isEmpty() && time.nanoseconds() - started < CREATE_TOPIC_TIMEOUT_NS) { time.sleep(sleepMs); sleepMs = Math.min(2 * sleepMs, MAX_SLEEP_MS); partitionInfos = consumer.partitionsFor(topic); } - if (partitionInfos == null) + if (partitionInfos.isEmpty()) throw new ConnectException("Could not look up partition metadata for offset backing store topic in" + " allotted period. This could indicate a connectivity issue, unavailable topic partitions, or if" + " this is your first use of the topic it may have taken too long to create."); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorTopicsIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorTopicsIntegrationTest.java index 75374a94819ac..8c4e156693121 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorTopicsIntegrationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorTopicsIntegrationTest.java @@ -19,6 +19,7 @@ import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.connect.runtime.distributed.DistributedConfig; import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException; @@ -40,7 +41,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; @@ -260,10 +260,11 @@ public void assertNoTopicStatusInStatusTopic() { Consumer verifiableConsumer = connect.kafka().createConsumer( Collections.singletonMap("group.id", "verifiable-consumer-group-0")); - List partitions = - Optional.ofNullable(verifiableConsumer.partitionsFor(statusTopic)) - .orElseThrow(() -> new AssertionError("Unable to retrieve partitions info for status topic")) - .stream() + List partitionInfos = verifiableConsumer.partitionsFor(statusTopic); + if (partitionInfos.isEmpty()) { + throw new AssertionError("Unable to retrieve partitions info for status topic"); + } + List partitions = partitionInfos.stream() .map(info -> new TopicPartition(info.topic(), info.partition())) .collect(Collectors.toList()); verifiableConsumer.assign(partitions); diff --git a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala index 2c0dc8c423ef7..6c6090f006dee 100755 --- a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala +++ b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala @@ -21,7 +21,6 @@ import java.time.{Duration, Instant} import java.util.Properties import com.fasterxml.jackson.dataformat.csv.CsvMapper import com.fasterxml.jackson.module.scala.DefaultScalaModule -import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper import kafka.utils._ import kafka.utils.Implicits._ import org.apache.kafka.clients.admin._ @@ -146,7 +145,7 @@ object ConsumerGroupCommand extends Logging { } // Example: CsvUtils().readerFor[CsvRecordWithoutGroup] private[admin] case class CsvUtils() { - val mapper = new CsvMapper with ScalaObjectMapper + val mapper = new CsvMapper mapper.registerModule(DefaultScalaModule) def readerFor[T <: CsvRecord : ClassTag] = { val schema = getSchema[T] diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala index 7eea0e2bbf656..89cadf4485851 100755 --- a/core/src/main/scala/kafka/cluster/Partition.scala +++ b/core/src/main/scala/kafka/cluster/Partition.scala @@ -18,7 +18,6 @@ package kafka.cluster import java.util.concurrent.locks.ReentrantReadWriteLock import java.util.Optional - import kafka.api.{ApiVersion, LeaderAndIsr} import kafka.common.UnexpectedAppendOffsetException import kafka.controller.{KafkaController, StateChangeLogger} @@ -1019,7 +1018,8 @@ class Partition(val topicPartition: TopicPartition, } } - def appendRecordsToLeader(records: MemoryRecords, origin: AppendOrigin, requiredAcks: Int): LogAppendInfo = { + def appendRecordsToLeader(records: MemoryRecords, origin: AppendOrigin, requiredAcks: Int, + requestLocal: RequestLocal): LogAppendInfo = { val (info, leaderHWIncremented) = inReadLock(leaderIsrUpdateLock) { leaderLogIfLocal match { case Some(leaderLog) => @@ -1033,7 +1033,7 @@ class Partition(val topicPartition: TopicPartition, } val info = leaderLog.appendAsLeader(records, leaderEpoch = this.leaderEpoch, origin, - interBrokerProtocolVersion) + interBrokerProtocolVersion, requestLocal) // we may need to increment high watermark since ISR could be down to 1 (info, maybeIncrementLeaderHW(leaderLog)) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala index bb06ca538f9dd..f3e170b81ac0c 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala @@ -18,7 +18,6 @@ package kafka.coordinator.group import java.util.Properties import java.util.concurrent.atomic.AtomicBoolean - import kafka.common.OffsetAndMetadata import kafka.log.LogConfig import kafka.message.ProducerCompressionCodec @@ -92,6 +91,7 @@ class GroupCoordinator(val brokerId: Int, props.put(LogConfig.CleanupPolicyProp, LogConfig.Compact) props.put(LogConfig.SegmentBytesProp, offsetConfig.offsetsTopicSegmentBytes.toString) props.put(LogConfig.CompressionTypeProp, ProducerCompressionCodec.name) + props } @@ -162,7 +162,8 @@ class GroupCoordinator(val brokerId: Int, sessionTimeoutMs: Int, protocolType: String, protocols: List[(String, Array[Byte])], - responseCallback: JoinCallback): Unit = { + responseCallback: JoinCallback, + requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { validateGroupStatus(groupId, ApiKeys.JOIN_GROUP).foreach { error => responseCallback(JoinGroupResult(memberId, error)) return @@ -194,7 +195,8 @@ class GroupCoordinator(val brokerId: Int, sessionTimeoutMs, protocolType, protocols, - responseCallback + responseCallback, + requestLocal ) } else { doCurrentMemberJoinGroup( @@ -230,7 +232,8 @@ class GroupCoordinator(val brokerId: Int, sessionTimeoutMs: Int, protocolType: String, protocols: List[(String, Array[Byte])], - responseCallback: JoinCallback + responseCallback: JoinCallback, + requestLocal: RequestLocal ): Unit = { group.inLock { if (group.is(Dead)) { @@ -255,9 +258,9 @@ class GroupCoordinator(val brokerId: Int, sessionTimeoutMs, protocolType, protocols, - responseCallback + responseCallback, + requestLocal ) - case None => doDynamicNewMemberJoinGroup( group, @@ -286,14 +289,15 @@ class GroupCoordinator(val brokerId: Int, sessionTimeoutMs: Int, protocolType: String, protocols: List[(String, Array[Byte])], - responseCallback: JoinCallback + responseCallback: JoinCallback, + requestLocal: RequestLocal ): Unit = { group.currentStaticMemberId(groupInstanceId) match { case Some(oldMemberId) => info(s"Static member with groupInstanceId=$groupInstanceId and unknown member id joins " + s"group ${group.groupId} in ${group.currentState} state. Replacing previously mapped " + s"member $oldMemberId with this groupInstanceId.") - updateStaticMemberAndRebalance(group, oldMemberId, newMemberId, groupInstanceId, protocols, responseCallback) + updateStaticMemberAndRebalance(group, oldMemberId, newMemberId, groupInstanceId, protocols, responseCallback, requestLocal) case None => info(s"Static member with groupInstanceId=$groupInstanceId and unknown member id joins " + @@ -474,7 +478,8 @@ class GroupCoordinator(val brokerId: Int, protocolName: Option[String], groupInstanceId: Option[String], groupAssignment: Map[String, Array[Byte]], - responseCallback: SyncCallback): Unit = { + responseCallback: SyncCallback, + requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { validateGroupStatus(groupId, ApiKeys.SYNC_GROUP) match { case Some(error) if error == Errors.COORDINATOR_LOAD_IN_PROGRESS => // The coordinator is loading, which means we've lost the state of the active rebalance and the @@ -489,7 +494,7 @@ class GroupCoordinator(val brokerId: Int, groupManager.getGroup(groupId) match { case None => responseCallback(SyncGroupResult(Errors.UNKNOWN_MEMBER_ID)) case Some(group) => doSyncGroup(group, generation, memberId, protocolType, protocolName, - groupInstanceId, groupAssignment, responseCallback) + groupInstanceId, groupAssignment, requestLocal, responseCallback) } } } @@ -535,6 +540,7 @@ class GroupCoordinator(val brokerId: Int, protocolName: Option[String], groupInstanceId: Option[String], groupAssignment: Map[String, Array[Byte]], + requestLocal: RequestLocal, responseCallback: SyncCallback): Unit = { group.inLock { val validationErrorOpt = validateSyncGroup( @@ -587,7 +593,7 @@ class GroupCoordinator(val brokerId: Int, } } } - }) + }, requestLocal) groupCompletedRebalanceSensor.record() } @@ -669,7 +675,8 @@ class GroupCoordinator(val brokerId: Int, } } - def handleDeleteGroups(groupIds: Set[String]): Map[String, Errors] = { + def handleDeleteGroups(groupIds: Set[String], + requestLocal: RequestLocal = RequestLocal.NoCaching): Map[String, Errors] = { val groupErrors = mutable.Map.empty[String, Errors] val groupsEligibleForDeletion = mutable.ArrayBuffer[GroupMetadata]() @@ -701,7 +708,8 @@ class GroupCoordinator(val brokerId: Int, } if (groupsEligibleForDeletion.nonEmpty) { - val offsetsRemoved = groupManager.cleanupGroupMetadata(groupsEligibleForDeletion, _.removeAllOffsets()) + val offsetsRemoved = groupManager.cleanupGroupMetadata(groupsEligibleForDeletion, requestLocal, + _.removeAllOffsets()) groupErrors ++= groupsEligibleForDeletion.map(_.groupId -> Errors.NONE).toMap info(s"The following groups were deleted: ${groupsEligibleForDeletion.map(_.groupId).mkString(", ")}. " + s"A total of $offsetsRemoved offsets were removed.") @@ -710,7 +718,8 @@ class GroupCoordinator(val brokerId: Int, groupErrors } - def handleDeleteOffsets(groupId: String, partitions: Seq[TopicPartition]): (Errors, Map[TopicPartition, Errors]) = { + def handleDeleteOffsets(groupId: String, partitions: Seq[TopicPartition], + requestLocal: RequestLocal): (Errors, Map[TopicPartition, Errors]) = { var groupError: Errors = Errors.NONE var partitionErrors: Map[TopicPartition, Errors] = Map() var partitionsEligibleForDeletion: Seq[TopicPartition] = Seq() @@ -748,9 +757,8 @@ class GroupCoordinator(val brokerId: Int, } if (partitionsEligibleForDeletion.nonEmpty) { - val offsetsRemoved = groupManager.cleanupGroupMetadata(Seq(group), group => { - group.removeOffsets(partitionsEligibleForDeletion) - }) + val offsetsRemoved = groupManager.cleanupGroupMetadata(Seq(group), requestLocal, + _.removeOffsets(partitionsEligibleForDeletion)) partitionErrors ++= partitionsEligibleForDeletion.map(_ -> Errors.NONE).toMap @@ -855,14 +863,16 @@ class GroupCoordinator(val brokerId: Int, groupInstanceId: Option[String], generationId: Int, offsetMetadata: immutable.Map[TopicPartition, OffsetAndMetadata], - responseCallback: immutable.Map[TopicPartition, Errors] => Unit): Unit = { + responseCallback: immutable.Map[TopicPartition, Errors] => Unit, + requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { validateGroupStatus(groupId, ApiKeys.TXN_OFFSET_COMMIT) match { case Some(error) => responseCallback(offsetMetadata.map { case (k, _) => k -> error }) case None => val group = groupManager.getGroup(groupId).getOrElse { groupManager.addGroup(new GroupMetadata(groupId, Empty, time)) } - doTxnCommitOffsets(group, memberId, groupInstanceId, generationId, producerId, producerEpoch, offsetMetadata, responseCallback) + doTxnCommitOffsets(group, memberId, groupInstanceId, generationId, producerId, producerEpoch, + offsetMetadata, requestLocal, responseCallback) } } @@ -871,7 +881,8 @@ class GroupCoordinator(val brokerId: Int, groupInstanceId: Option[String], generationId: Int, offsetMetadata: immutable.Map[TopicPartition, OffsetAndMetadata], - responseCallback: immutable.Map[TopicPartition, Errors] => Unit): Unit = { + responseCallback: immutable.Map[TopicPartition, Errors] => Unit, + requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { validateGroupStatus(groupId, ApiKeys.OFFSET_COMMIT) match { case Some(error) => responseCallback(offsetMetadata.map { case (k, _) => k -> error }) case None => @@ -880,14 +891,16 @@ class GroupCoordinator(val brokerId: Int, if (generationId < 0) { // the group is not relying on Kafka for group management, so allow the commit val group = groupManager.addGroup(new GroupMetadata(groupId, Empty, time)) - doCommitOffsets(group, memberId, groupInstanceId, generationId, offsetMetadata, responseCallback) + doCommitOffsets(group, memberId, groupInstanceId, generationId, offsetMetadata, + responseCallback, requestLocal) } else { // or this is a request coming from an older generation. either way, reject the commit responseCallback(offsetMetadata.map { case (k, _) => k -> Errors.ILLEGAL_GENERATION }) } case Some(group) => - doCommitOffsets(group, memberId, groupInstanceId, generationId, offsetMetadata, responseCallback) + doCommitOffsets(group, memberId, groupInstanceId, generationId, offsetMetadata, + responseCallback, requestLocal) } } } @@ -907,6 +920,7 @@ class GroupCoordinator(val brokerId: Int, producerId: Long, producerEpoch: Short, offsetMetadata: immutable.Map[TopicPartition, OffsetAndMetadata], + requestLocal: RequestLocal, responseCallback: immutable.Map[TopicPartition, Errors] => Unit): Unit = { group.inLock { val validationErrorOpt = validateOffsetCommit( @@ -920,7 +934,8 @@ class GroupCoordinator(val brokerId: Int, if (validationErrorOpt.isDefined) { responseCallback(offsetMetadata.map { case (k, _) => k -> validationErrorOpt.get }) } else { - groupManager.storeOffsets(group, memberId, offsetMetadata, responseCallback, producerId, producerEpoch) + groupManager.storeOffsets(group, memberId, offsetMetadata, responseCallback, producerId, + producerEpoch, requestLocal) } } } @@ -963,7 +978,8 @@ class GroupCoordinator(val brokerId: Int, groupInstanceId: Option[String], generationId: Int, offsetMetadata: immutable.Map[TopicPartition, OffsetAndMetadata], - responseCallback: immutable.Map[TopicPartition, Errors] => Unit): Unit = { + responseCallback: immutable.Map[TopicPartition, Errors] => Unit, + requestLocal: RequestLocal): Unit = { group.inLock { val validationErrorOpt = validateOffsetCommit( group, @@ -985,7 +1001,7 @@ class GroupCoordinator(val brokerId: Int, // on heartbeat response to eventually notify the rebalance in progress signal to the consumer val member = group.get(memberId) completeAndScheduleNextHeartbeatExpiration(group, member) - groupManager.storeOffsets(group, memberId, offsetMetadata, responseCallback) + groupManager.storeOffsets(group, memberId, offsetMetadata, responseCallback, requestLocal = requestLocal) case CompletingRebalance => // We should not receive a commit request if the group has not completed rebalance; @@ -1041,10 +1057,9 @@ class GroupCoordinator(val brokerId: Int, } } - def handleDeletedPartitions(topicPartitions: Seq[TopicPartition]): Unit = { - val offsetsRemoved = groupManager.cleanupGroupMetadata(groupManager.currentGroups, group => { - group.removeOffsets(topicPartitions) - }) + def handleDeletedPartitions(topicPartitions: Seq[TopicPartition], requestLocal: RequestLocal): Unit = { + val offsetsRemoved = groupManager.cleanupGroupMetadata(groupManager.currentGroups, requestLocal, + _.removeOffsets(topicPartitions)) info(s"Removed $offsetsRemoved offsets associated with deleted partitions: ${topicPartitions.mkString(", ")}.") } @@ -1235,7 +1250,8 @@ class GroupCoordinator(val brokerId: Int, newMemberId: String, groupInstanceId: String, protocols: List[(String, Array[Byte])], - responseCallback: JoinCallback): Unit = { + responseCallback: JoinCallback, + requestLocal: RequestLocal): Unit = { val currentLeader = group.leaderOrNull val member = group.replaceStaticMember(groupInstanceId, oldMemberId, newMemberId) // Heartbeat of old member id will expire without effect since the group no longer contains that member id. @@ -1287,7 +1303,7 @@ class GroupCoordinator(val brokerId: Int, leaderId = currentLeader, error = Errors.NONE)) } - }) + }, requestLocal) } else { maybePrepareRebalance(group, s"Group's selectedProtocol will change because static member ${member.memberId} with instance id $groupInstanceId joined with change of protocol") } @@ -1411,7 +1427,7 @@ class GroupCoordinator(val brokerId: Int, // This should be safe since there are no active members in an empty generation, so we just warn. warn(s"Failed to write empty metadata for group ${group.groupId}: ${error.message}") } - }) + }, RequestLocal.NoCaching) } else { info(s"Stabilized group ${group.groupId} generation ${group.generationId} " + s"(${Topic.GROUP_METADATA_TOPIC_NAME}-${partitionFor(group.groupId)}) with ${group.size} members") diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index c054234abb082..d3d911d70b0cc 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -24,14 +24,13 @@ import java.util.Optional import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.locks.ReentrantLock - import com.yammer.metrics.core.Gauge import kafka.api.{ApiVersion, KAFKA_0_10_1_IV0, KAFKA_2_1_IV0, KAFKA_2_1_IV1, KAFKA_2_3_IV0} import kafka.common.OffsetAndMetadata import kafka.internals.generated.{GroupMetadataValue, OffsetCommitKey, OffsetCommitValue, GroupMetadataKey => GroupMetadataKeyData} import kafka.log.AppendOrigin import kafka.metrics.KafkaMetricsGroup -import kafka.server.{FetchLogEnd, ReplicaManager} +import kafka.server.{FetchLogEnd, ReplicaManager, RequestLocal} import kafka.utils.CoreUtils.inLock import kafka.utils.Implicits._ import kafka.utils._ @@ -240,7 +239,8 @@ class GroupMetadataManager(brokerId: Int, def storeGroup(group: GroupMetadata, groupAssignment: Map[String, Array[Byte]], - responseCallback: Errors => Unit): Unit = { + responseCallback: Errors => Unit, + requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { getMagic(partitionFor(group.groupId)) match { case Some(magicValue) => // We always use CREATE_TIME, like the producer. The conversion to LOG_APPEND_TIME (if necessary) happens automatically. @@ -310,7 +310,7 @@ class GroupMetadataManager(brokerId: Int, responseCallback(responseError) } - appendForGroup(group, groupMetadataRecords, putCacheCallback) + appendForGroup(group, groupMetadataRecords, requestLocal, putCacheCallback) case None => responseCallback(Errors.NOT_COORDINATOR) @@ -320,6 +320,7 @@ class GroupMetadataManager(brokerId: Int, private def appendForGroup(group: GroupMetadata, records: Map[TopicPartition, MemoryRecords], + requestLocal: RequestLocal, callback: Map[TopicPartition, PartitionResponse] => Unit): Unit = { // call replica manager to append the group message replicaManager.appendRecords( @@ -329,7 +330,8 @@ class GroupMetadataManager(brokerId: Int, origin = AppendOrigin.Coordinator, entriesPerPartition = records, delayedProduceLock = Some(group.lock), - responseCallback = callback) + responseCallback = callback, + requestLocal = requestLocal) } /** @@ -340,7 +342,8 @@ class GroupMetadataManager(brokerId: Int, offsetMetadata: immutable.Map[TopicPartition, OffsetAndMetadata], responseCallback: immutable.Map[TopicPartition, Errors] => Unit, producerId: Long = RecordBatch.NO_PRODUCER_ID, - producerEpoch: Short = RecordBatch.NO_PRODUCER_EPOCH): Unit = { + producerEpoch: Short = RecordBatch.NO_PRODUCER_EPOCH, + requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { // first filter out partitions with offset metadata size exceeding limit val filteredOffsetMetadata = offsetMetadata.filter { case (_, offsetAndMetadata) => validateOffsetMetadataLength(offsetAndMetadata.metadata) @@ -467,7 +470,7 @@ class GroupMetadataManager(brokerId: Int, } } - appendForGroup(group, entries, putCacheCallback) + appendForGroup(group, entries, requestLocal, putCacheCallback) case None => val commitStatus = offsetMetadata.map { case (topicPartition, _) => @@ -781,9 +784,8 @@ class GroupMetadataManager(brokerId: Int, // visible for testing private[group] def cleanupGroupMetadata(): Unit = { val currentTimestamp = time.milliseconds() - val numOffsetsRemoved = cleanupGroupMetadata(groupMetadataCache.values, group => { - group.removeExpiredOffsets(currentTimestamp, config.offsetsRetentionMs) - }) + val numOffsetsRemoved = cleanupGroupMetadata(groupMetadataCache.values, RequestLocal.NoCaching, + _.removeExpiredOffsets(currentTimestamp, config.offsetsRetentionMs)) offsetExpiredSensor.record(numOffsetsRemoved) if (numOffsetsRemoved > 0) info(s"Removed $numOffsetsRemoved expired offsets in ${time.milliseconds() - currentTimestamp} milliseconds.") @@ -796,7 +798,8 @@ class GroupMetadataManager(brokerId: Int, * a group lock is held, therefore there is no need for the caller to also obtain a group lock. * @return The cumulative number of offsets removed */ - def cleanupGroupMetadata(groups: Iterable[GroupMetadata], selector: GroupMetadata => Map[TopicPartition, OffsetAndMetadata]): Int = { + def cleanupGroupMetadata(groups: Iterable[GroupMetadata], requestLocal: RequestLocal, + selector: GroupMetadata => Map[TopicPartition, OffsetAndMetadata]): Int = { var offsetsRemoved = 0 groups.foreach { group => @@ -843,7 +846,8 @@ class GroupMetadataManager(brokerId: Int, // do not need to require acks since even if the tombstone is lost, // it will be appended again in the next purge cycle val records = MemoryRecords.withRecords(magicValue, 0L, compressionType, timestampType, tombstones.toArray: _*) - partition.appendRecordsToLeader(records, origin = AppendOrigin.Coordinator, requiredAcks = 0) + partition.appendRecordsToLeader(records, origin = AppendOrigin.Coordinator, requiredAcks = 0, + requestLocal = requestLocal) offsetsRemoved += removedOffsets.size trace(s"Successfully appended ${tombstones.size} tombstones to $appendPartition for expired/deleted " + diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala index 543e9c85c36d5..78983c16cbf67 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala @@ -18,8 +18,7 @@ package kafka.coordinator.transaction import java.util.Properties import java.util.concurrent.atomic.AtomicBoolean - -import kafka.server.{KafkaConfig, MetadataCache, ReplicaManager} +import kafka.server.{KafkaConfig, MetadataCache, ReplicaManager, RequestLocal} import kafka.utils.{Logging, Scheduler} import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.internals.Topic @@ -104,7 +103,8 @@ class TransactionCoordinator(brokerId: Int, def handleInitProducerId(transactionalId: String, transactionTimeoutMs: Int, expectedProducerIdAndEpoch: Option[ProducerIdAndEpoch], - responseCallback: InitProducerIdCallback): Unit = { + responseCallback: InitProducerIdCallback, + requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { if (transactionalId == null) { // if the transactional id is null, then always blindly accept the request @@ -167,7 +167,8 @@ class TransactionCoordinator(brokerId: Int, newMetadata.producerEpoch, TransactionResult.ABORT, isFromClient = false, - sendRetriableErrorCallback) + sendRetriableErrorCallback, + requestLocal) } else { def sendPidResponseCallback(error: Errors): Unit = { if (error == Errors.NONE) { @@ -181,7 +182,8 @@ class TransactionCoordinator(brokerId: Int, } } - txnManager.appendTransactionToLog(transactionalId, coordinatorEpoch, newMetadata, sendPidResponseCallback) + txnManager.appendTransactionToLog(transactionalId, coordinatorEpoch, newMetadata, + sendPidResponseCallback, requestLocal = requestLocal) } } } @@ -320,7 +322,8 @@ class TransactionCoordinator(brokerId: Int, producerId: Long, producerEpoch: Short, partitions: collection.Set[TopicPartition], - responseCallback: AddPartitionsCallback): Unit = { + responseCallback: AddPartitionsCallback, + requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { if (transactionalId == null || transactionalId.isEmpty) { debug(s"Returning ${Errors.INVALID_REQUEST} error code to client for $transactionalId's AddPartitions request") responseCallback(Errors.INVALID_REQUEST) @@ -360,7 +363,8 @@ class TransactionCoordinator(brokerId: Int, responseCallback(err) case Right((coordinatorEpoch, newMetadata)) => - txnManager.appendTransactionToLog(transactionalId, coordinatorEpoch, newMetadata, responseCallback) + txnManager.appendTransactionToLog(transactionalId, coordinatorEpoch, newMetadata, + responseCallback, requestLocal = requestLocal) } } } @@ -413,13 +417,15 @@ class TransactionCoordinator(brokerId: Int, producerId: Long, producerEpoch: Short, txnMarkerResult: TransactionResult, - responseCallback: EndTxnCallback): Unit = { + responseCallback: EndTxnCallback, + requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { endTransaction(transactionalId, producerId, producerEpoch, txnMarkerResult, isFromClient = true, - responseCallback) + responseCallback, + requestLocal) } private def endTransaction(transactionalId: String, @@ -427,7 +433,8 @@ class TransactionCoordinator(brokerId: Int, producerEpoch: Short, txnMarkerResult: TransactionResult, isFromClient: Boolean, - responseCallback: EndTxnCallback): Unit = { + responseCallback: EndTxnCallback, + requestLocal: RequestLocal): Unit = { var isEpochFence = false if (transactionalId == null || transactionalId.isEmpty) responseCallback(Errors.INVALID_REQUEST) @@ -586,7 +593,8 @@ class TransactionCoordinator(brokerId: Int, } } - txnManager.appendTransactionToLog(transactionalId, coordinatorEpoch, newMetadata, sendTxnMarkersCallback) + txnManager.appendTransactionToLog(transactionalId, coordinatorEpoch, newMetadata, + sendTxnMarkersCallback, requestLocal = requestLocal) } } } @@ -643,7 +651,8 @@ class TransactionCoordinator(brokerId: Int, txnTransitMetadata.producerEpoch, TransactionResult.ABORT, isFromClient = false, - onComplete(txnIdAndPidEpoch)) + onComplete(txnIdAndPidEpoch), + RequestLocal.NoCaching) } } } diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerChannelManager.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerChannelManager.scala index 5e22fb7340364..62c70d91121db 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerChannelManager.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerChannelManager.scala @@ -19,11 +19,10 @@ package kafka.coordinator.transaction import java.util import java.util.concurrent.{BlockingQueue, ConcurrentHashMap, LinkedBlockingQueue} - import kafka.api.KAFKA_2_8_IV0 import kafka.common.{InterBrokerSendThread, RequestAndCompletionHandler} import kafka.metrics.KafkaMetricsGroup -import kafka.server.{KafkaConfig, MetadataCache} +import kafka.server.{KafkaConfig, MetadataCache, RequestLocal} import kafka.utils.Implicits._ import kafka.utils.{CoreUtils, Logging} import org.apache.kafka.clients._ @@ -330,8 +329,8 @@ class TransactionMarkerChannelManager( throw new IllegalStateException(errorMsg) } - txnStateManager.appendTransactionToLog(txnLogAppend.transactionalId, txnLogAppend.coordinatorEpoch, txnLogAppend.newMetadata, appendCallback, - _ == Errors.COORDINATOR_NOT_AVAILABLE) + txnStateManager.appendTransactionToLog(txnLogAppend.transactionalId, txnLogAppend.coordinatorEpoch, + txnLogAppend.newMetadata, appendCallback, _ == Errors.COORDINATOR_NOT_AVAILABLE, RequestLocal.NoCaching) } def addTxnMarkersToBrokerQueue(transactionalId: String, diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala index 61fad952dc44f..25580f27a6b74 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala @@ -21,10 +21,9 @@ import java.util.Properties import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.locks.ReentrantReadWriteLock - import kafka.log.{AppendOrigin, LogConfig} import kafka.message.UncompressedCodec -import kafka.server.{Defaults, FetchLogEnd, ReplicaManager} +import kafka.server.{Defaults, FetchLogEnd, ReplicaManager, RequestLocal} import kafka.utils.CoreUtils.{inReadLock, inWriteLock} import kafka.utils.{Logging, Pool, Scheduler} import kafka.utils.Implicits._ @@ -209,7 +208,8 @@ class TransactionStateManager(brokerId: Int, internalTopicsAllowed = true, origin = AppendOrigin.Coordinator, recordsPerPartition, - removeFromCacheCallback) + removeFromCacheCallback, + requestLocal = RequestLocal.NoCaching) } }, delay = config.removeExpiredTransactionalIdsIntervalMs, period = config.removeExpiredTransactionalIdsIntervalMs) @@ -526,7 +526,8 @@ class TransactionStateManager(brokerId: Int, coordinatorEpoch: Int, newMetadata: TxnTransitMetadata, responseCallback: Errors => Unit, - retryOnError: Errors => Boolean = _ => false): Unit = { + retryOnError: Errors => Boolean = _ => false, + requestLocal: RequestLocal): Unit = { // generate the message for this transaction metadata val keyBytes = TransactionLog.keyToBytes(transactionalId) @@ -679,7 +680,8 @@ class TransactionStateManager(brokerId: Int, internalTopicsAllowed = true, origin = AppendOrigin.Coordinator, recordsPerPartition, - updateCacheCallback) + updateCacheCallback, + requestLocal = requestLocal) trace(s"Appending new metadata $newMetadata for transaction id $transactionalId with coordinator epoch $coordinatorEpoch to the local transaction log") } diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala index ef0d6aed5f0d8..b49bfb46dbdff 100644 --- a/core/src/main/scala/kafka/log/Log.scala +++ b/core/src/main/scala/kafka/log/Log.scala @@ -18,15 +18,12 @@ package kafka.log import java.io.{File, IOException} -import java.lang.{Long => JLong} import java.nio.file.Files import java.text.NumberFormat -import java.util.Map.{Entry => JEntry} import java.util.Optional import java.util.concurrent.atomic._ import java.util.concurrent.TimeUnit import java.util.regex.Pattern - import kafka.api.{ApiVersion, KAFKA_0_10_0_IV0} import kafka.common.{LongRef, OffsetsOutOfOrderException, UnexpectedAppendOffsetException} import kafka.log.AppendOrigin.RaftLeader @@ -34,7 +31,7 @@ import kafka.message.{BrokerCompressionCodec, CompressionCodec, NoCompressionCod import kafka.metrics.KafkaMetricsGroup import kafka.server.checkpoints.LeaderEpochCheckpointFile import kafka.server.epoch.LeaderEpochFileCache -import kafka.server.{BrokerTopicStats, FetchDataInfo, FetchHighWatermark, FetchIsolation, FetchLogEnd, FetchTxnCommitted, LogDirFailureChannel, LogOffsetMetadata, OffsetAndEpoch, PartitionMetadataFile} +import kafka.server.{BrokerTopicStats, FetchDataInfo, FetchHighWatermark, FetchIsolation, FetchLogEnd, FetchTxnCommitted, LogDirFailureChannel, LogOffsetMetadata, OffsetAndEpoch, PartitionMetadataFile, RequestLocal} import kafka.utils._ import org.apache.kafka.common.errors._ import org.apache.kafka.common.message.{DescribeProducersResponseData, FetchResponseData} @@ -272,7 +269,7 @@ class Log(@volatile private var _dir: File, @volatile var leaderEpochCache: Option[LeaderEpochFileCache], val producerStateManager: ProducerStateManager, logDirFailureChannel: LogDirFailureChannel, - @volatile var topicId: Option[Uuid], + @volatile private var _topicId: Option[Uuid], val keepPartitionMetadataFile: Boolean) extends Logging with KafkaMetricsGroup { import kafka.log.Log._ @@ -324,20 +321,26 @@ class Log(@volatile private var _dir: File, // write to the partition metadata file. // Ensure we do not try to assign a provided topicId that is inconsistent with the ID on file. if (partitionMetadataFile.exists()) { - if (!keepPartitionMetadataFile) - partitionMetadataFile.delete() - else { + if (keepPartitionMetadataFile) { val fileTopicId = partitionMetadataFile.read().topicId - if (topicId.isDefined && !topicId.contains(fileTopicId)) + if (_topicId.isDefined && !_topicId.contains(fileTopicId)) throw new InconsistentTopicIdException(s"Tried to assign topic ID $topicId to log for topic partition $topicPartition," + s"but log already contained topic ID $fileTopicId") - topicId = Some(fileTopicId) + _topicId = Some(fileTopicId) + } else { + try partitionMetadataFile.delete() + catch { + case e: IOException => + error(s"Error while trying to delete partition metadata file ${partitionMetadataFile}", e) + } } } else if (keepPartitionMetadataFile) { - topicId.foreach(partitionMetadataFile.write) + _topicId.foreach(partitionMetadataFile.write) } } + def topicId: Option[Uuid] = _topicId + def dir: File = _dir def parentDir: String = _parentDir @@ -553,11 +556,11 @@ class Log(@volatile private var _dir: File, /** Only used for ZK clusters when we update and start using topic IDs on existing topics */ def assignTopicId(topicId: Uuid): Unit = { partitionMetadataFile.write(topicId) - this.topicId = Some(topicId) + _topicId = Some(topicId) } private def initializeLeaderEpochCache(): Unit = lock synchronized { - leaderEpochCache = Log.maybeCreateLeaderEpochCache(dir, topicPartition, logDirFailureChannel, recordVersion) + leaderEpochCache = Log.maybeCreateLeaderEpochCache(dir, topicPartition, logDirFailureChannel, recordVersion, logIdent) } private def updateLogEndOffset(offset: Long): Unit = { @@ -592,7 +595,7 @@ class Log(@volatile private var _dir: File, producerStateManager: ProducerStateManager): Unit = lock synchronized { checkIfMemoryMappedBufferClosed() Log.rebuildProducerState(producerStateManager, segments, logStartOffset, lastOffset, recordVersion, time, - reloadFromCleanShutdown = false) + reloadFromCleanShutdown = false, logIdent) } def activeProducers: Seq[DescribeProducersResponseData.ProducerState] = { @@ -689,15 +692,17 @@ class Log(@volatile private var _dir: File, * @param records The records to append * @param origin Declares the origin of the append which affects required validations * @param interBrokerProtocolVersion Inter-broker message protocol version + * @param requestLocal request local instance * @throws KafkaStorageException If the append fails due to an I/O error. * @return Information about the appended messages including the first and last offset. */ def appendAsLeader(records: MemoryRecords, leaderEpoch: Int, origin: AppendOrigin = AppendOrigin.Client, - interBrokerProtocolVersion: ApiVersion = ApiVersion.latestVersion): LogAppendInfo = { + interBrokerProtocolVersion: ApiVersion = ApiVersion.latestVersion, + requestLocal: RequestLocal = RequestLocal.NoCaching): LogAppendInfo = { val validateAndAssignOffsets = origin != AppendOrigin.RaftLeader - append(records, origin, interBrokerProtocolVersion, validateAndAssignOffsets, leaderEpoch, ignoreRecordSize = false) + append(records, origin, interBrokerProtocolVersion, validateAndAssignOffsets, leaderEpoch, Some(requestLocal), ignoreRecordSize = false) } /** @@ -713,6 +718,7 @@ class Log(@volatile private var _dir: File, interBrokerProtocolVersion = ApiVersion.latestVersion, validateAndAssignOffsets = false, leaderEpoch = -1, + None, // disable to check the validation of record size since the record is already accepted by leader. ignoreRecordSize = true) } @@ -728,6 +734,7 @@ class Log(@volatile private var _dir: File, * @param interBrokerProtocolVersion Inter-broker message protocol version * @param validateAndAssignOffsets Should the log assign offsets to this message set or blindly apply what it is given * @param leaderEpoch The partition's leader epoch which will be applied to messages when offsets are assigned on the leader + * @param requestLocal The request local instance if assignOffsets is true * @param ignoreRecordSize true to skip validation of record size. * @throws KafkaStorageException If the append fails due to an I/O error. * @throws OffsetsOutOfOrderException If out of order offsets found in 'records' @@ -739,6 +746,7 @@ class Log(@volatile private var _dir: File, interBrokerProtocolVersion: ApiVersion, validateAndAssignOffsets: Boolean, leaderEpoch: Int, + requestLocal: Option[RequestLocal], ignoreRecordSize: Boolean): LogAppendInfo = { val appendInfo = analyzeAndValidateRecords(records, origin, ignoreRecordSize, leaderEpoch) @@ -774,7 +782,9 @@ class Log(@volatile private var _dir: File, leaderEpoch, origin, interBrokerProtocolVersion, - brokerTopicStats) + brokerTopicStats, + requestLocal.getOrElse(throw new IllegalArgumentException( + "requestLocal should be defined if assignOffsets is true"))) } catch { case e: IOException => throw new KafkaException(s"Error validating messages while appending to log $name", e) @@ -1181,10 +1191,10 @@ class Log(@volatile private var _dir: File, // We create the local variables to avoid race conditions with updates to the log. val endOffsetMetadata = nextOffsetMetadata val endOffset = endOffsetMetadata.messageOffset - var segmentEntryOpt = segments.floorEntry(startOffset) + var segmentOpt = segments.floorSegment(startOffset) // return error on attempt to read beyond the log end offset or read below log start offset - if (startOffset > endOffset || segmentEntryOpt.isEmpty || startOffset < logStartOffset) + if (startOffset > endOffset || segmentOpt.isEmpty || startOffset < logStartOffset) throw new OffsetOutOfRangeException(s"Received request for offset $startOffset for partition $topicPartition, " + s"but we only have log segments in the range $logStartOffset to $endOffset.") @@ -1202,12 +1212,10 @@ class Log(@volatile private var _dir: File, // Do the read on the segment with a base offset less than the target offset // but if that segment doesn't contain any messages with an offset greater than that // continue to read from successive segments until we get some messages or we reach the end of the log - var done = segmentEntryOpt.isEmpty var fetchDataInfo: FetchDataInfo = null - while (!done) { - val segmentEntry = segmentEntryOpt.get - val baseOffset = segmentEntry.getKey - val segment = segmentEntry.getValue + while (fetchDataInfo == null && segmentOpt.isDefined) { + val segment = segmentOpt.get + val baseOffset = segment.baseOffset val maxPosition = // Use the max offset position if it is on this segment; otherwise, the segment size is the limit. @@ -1217,10 +1225,8 @@ class Log(@volatile private var _dir: File, fetchDataInfo = segment.read(startOffset, maxLength, maxPosition, minOneMessage) if (fetchDataInfo != null) { if (includeAbortedTxns) - fetchDataInfo = addAbortedTransactions(startOffset, segmentEntry, fetchDataInfo) - } else segmentEntryOpt = segments.higherEntry(baseOffset) - - done = fetchDataInfo != null || segmentEntryOpt.isEmpty + fetchDataInfo = addAbortedTransactions(startOffset, segment, fetchDataInfo) + } else segmentOpt = segments.higherSegment(baseOffset) } if (fetchDataInfo != null) fetchDataInfo @@ -1235,25 +1241,25 @@ class Log(@volatile private var _dir: File, } private[log] def collectAbortedTransactions(startOffset: Long, upperBoundOffset: Long): List[AbortedTxn] = { - val segmentEntryOpt = segments.floorEntry(startOffset) + val segmentEntry = segments.floorSegment(startOffset) val allAbortedTxns = ListBuffer.empty[AbortedTxn] def accumulator(abortedTxns: List[AbortedTxn]): Unit = allAbortedTxns ++= abortedTxns - collectAbortedTransactions(logStartOffset, upperBoundOffset, segmentEntryOpt.get, accumulator) + segmentEntry.foreach(segment => collectAbortedTransactions(logStartOffset, upperBoundOffset, segment, accumulator)) allAbortedTxns.toList } - private def addAbortedTransactions(startOffset: Long, segmentEntry: JEntry[JLong, LogSegment], + private def addAbortedTransactions(startOffset: Long, segment: LogSegment, fetchInfo: FetchDataInfo): FetchDataInfo = { val fetchSize = fetchInfo.records.sizeInBytes val startOffsetPosition = OffsetPosition(fetchInfo.fetchOffsetMetadata.messageOffset, fetchInfo.fetchOffsetMetadata.relativePositionInSegment) - val upperBoundOffset = segmentEntry.getValue.fetchUpperBoundOffset(startOffsetPosition, fetchSize).getOrElse { - segments.higherSegment(segmentEntry.getKey).map(_.baseOffset).getOrElse(logEndOffset) + val upperBoundOffset = segment.fetchUpperBoundOffset(startOffsetPosition, fetchSize).getOrElse { + segments.higherSegment(segment.baseOffset).map(_.baseOffset).getOrElse(logEndOffset) } val abortedTransactions = ListBuffer.empty[FetchResponseData.AbortedTransaction] def accumulator(abortedTxns: List[AbortedTxn]): Unit = abortedTransactions ++= abortedTxns.map(_.asAbortedTransaction) - collectAbortedTransactions(startOffset, upperBoundOffset, segmentEntry, accumulator) + collectAbortedTransactions(startOffset, upperBoundOffset, segment, accumulator) FetchDataInfo(fetchOffsetMetadata = fetchInfo.fetchOffsetMetadata, records = fetchInfo.records, @@ -1262,17 +1268,17 @@ class Log(@volatile private var _dir: File, } private def collectAbortedTransactions(startOffset: Long, upperBoundOffset: Long, - startingSegmentEntry: JEntry[JLong, LogSegment], + startingSegment: LogSegment, accumulator: List[AbortedTxn] => Unit): Unit = { - var segmentEntryOpt = Option(startingSegmentEntry) + val higherSegments = segments.higherSegments(startingSegment.baseOffset).iterator + var segmentEntryOpt = Option(startingSegment) while (segmentEntryOpt.isDefined) { - val baseOffset = segmentEntryOpt.get.getKey - val segment = segmentEntryOpt.get.getValue + val segment = segmentEntryOpt.get val searchResult = segment.collectAbortedTxns(startOffset, upperBoundOffset) accumulator(searchResult.abortedTransactions) if (searchResult.isComplete) return - segmentEntryOpt = segments.higherEntry(baseOffset) + segmentEntryOpt = nextOption(higherSegments) } } @@ -1438,23 +1444,23 @@ class Log(@volatile private var _dir: File, Seq.empty } else { val deletable = ArrayBuffer.empty[LogSegment] - var segmentEntryOpt = segments.firstEntry - while (segmentEntryOpt.isDefined) { - val segmentEntry = segmentEntryOpt.get - val segment = segmentEntry.getValue - val nextSegmentEntryOpt = segments.higherEntry(segmentEntry.getKey) - val (nextSegment, upperBoundOffset, isLastSegmentAndEmpty) = - nextSegmentEntryOpt.map { - entry => (entry.getValue, entry.getValue.baseOffset, false) + val segmentsIterator = segments.values.iterator + var segmentOpt = nextOption(segmentsIterator) + while (segmentOpt.isDefined) { + val segment = segmentOpt.get + val nextSegmentOpt = nextOption(segmentsIterator) + val (upperBoundOffset: Long, isLastSegmentAndEmpty: Boolean) = + nextSegmentOpt.map { + nextSegment => (nextSegment.baseOffset, false) }.getOrElse { - (null, logEndOffset, segment.size == 0) + (logEndOffset, segment.size == 0) } - if (highWatermark >= upperBoundOffset && predicate(segment, Option(nextSegment)) && !isLastSegmentAndEmpty) { + if (highWatermark >= upperBoundOffset && predicate(segment, nextSegmentOpt) && !isLastSegmentAndEmpty) { deletable += segment - segmentEntryOpt = nextSegmentEntryOpt + segmentOpt = nextSegmentOpt } else { - segmentEntryOpt = Option.empty + segmentOpt = Option.empty } } deletable @@ -1888,14 +1894,14 @@ class Log(@volatile private var _dir: File, private def deleteSegmentFiles(segments: Iterable[LogSegment], asyncDelete: Boolean, deleteProducerStateSnapshots: Boolean = true): Unit = { Log.deleteSegmentFiles(segments, asyncDelete, deleteProducerStateSnapshots, dir, topicPartition, - config, scheduler, logDirFailureChannel, producerStateManager) + config, scheduler, logDirFailureChannel, producerStateManager, this.logIdent) } private[log] def replaceSegments(newSegments: Seq[LogSegment], oldSegments: Seq[LogSegment], isRecoveredSwapFile: Boolean = false): Unit = { lock synchronized { checkIfMemoryMappedBufferClosed() Log.replaceSegments(segments, newSegments, oldSegments, isRecoveredSwapFile, dir, topicPartition, - config, scheduler, logDirFailureChannel, producerStateManager) + config, scheduler, logDirFailureChannel, producerStateManager, this.logIdent) } } @@ -1937,7 +1943,7 @@ class Log(@volatile private var _dir: File, } private[log] def splitOverflowedSegment(segment: LogSegment): List[LogSegment] = lock synchronized { - Log.splitOverflowedSegment(segment, segments, dir, topicPartition, config, scheduler, logDirFailureChannel, producerStateManager) + Log.splitOverflowedSegment(segment, segments, dir, topicPartition, config, scheduler, logDirFailureChannel, producerStateManager, this.logIdent) } } @@ -2005,8 +2011,13 @@ object Log extends Logging { Files.createDirectories(dir.toPath) val topicPartition = Log.parseTopicPartitionName(dir) val segments = new LogSegments(topicPartition) - val leaderEpochCache = Log.maybeCreateLeaderEpochCache(dir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion) - val producerStateManager = new ProducerStateManager(topicPartition, dir, maxProducerIdExpirationMs) + val leaderEpochCache = Log.maybeCreateLeaderEpochCache( + dir, + topicPartition, + logDirFailureChannel, + config.messageFormatVersion.recordVersion, + s"[Log partition=$topicPartition, dir=${dir.getParent}] ") + val producerStateManager = new ProducerStateManager(topicPartition, dir, maxProducerIdExpirationMs, time) val offsets = LogLoader.load(LoadLogParams( dir, topicPartition, @@ -2226,12 +2237,14 @@ object Log extends Logging { * @param topicPartition The topic partition * @param logDirFailureChannel The LogDirFailureChannel to asynchronously handle log dir failure * @param recordVersion The record version + * @param logPrefix The logging prefix * @return The new LeaderEpochFileCache instance (if created), none otherwise */ def maybeCreateLeaderEpochCache(dir: File, topicPartition: TopicPartition, logDirFailureChannel: LogDirFailureChannel, - recordVersion: RecordVersion): Option[LeaderEpochFileCache] = { + recordVersion: RecordVersion, + logPrefix: String): Option[LeaderEpochFileCache] = { val leaderEpochFile = LeaderEpochCheckpointFile.newFile(dir) def newLeaderEpochFileCache(): LeaderEpochFileCache = { @@ -2246,7 +2259,7 @@ object Log extends Logging { None if (currentCache.exists(_.nonEmpty)) - warn(s"Deleting non-empty leader epoch cache due to incompatible message format $recordVersion") + warn(s"${logPrefix}Deleting non-empty leader epoch cache due to incompatible message format $recordVersion") Files.deleteIfExists(leaderEpochFile.toPath) None @@ -2293,6 +2306,7 @@ object Log extends Logging { * @param logDirFailureChannel The LogDirFailureChannel to asynchronously handle log dir failure * @param producerStateManager The ProducerStateManager instance (if any) containing state associated * with the existingSegments + * @param logPrefix The logging prefix */ private[log] def replaceSegments(existingSegments: LogSegments, newSegments: Seq[LogSegment], @@ -2303,7 +2317,8 @@ object Log extends Logging { config: LogConfig, scheduler: Scheduler, logDirFailureChannel: LogDirFailureChannel, - producerStateManager: ProducerStateManager): Unit = { + producerStateManager: ProducerStateManager, + logPrefix: String): Unit = { val sortedNewSegments = newSegments.sortBy(_.baseOffset) // Some old segments may have been removed from index and scheduled for async deletion after the caller reads segments // but before this method is executed. We want to filter out those segments to avoid calling asyncDeleteSegment() @@ -2332,7 +2347,8 @@ object Log extends Logging { config, scheduler, logDirFailureChannel, - producerStateManager) + producerStateManager, + logPrefix) } // okay we are safe now, remove the swap suffix sortedNewSegments.foreach(_.changeFileSuffixes(Log.SwapFileSuffix, "")) @@ -2359,7 +2375,7 @@ object Log extends Logging { * @param logDirFailureChannel The LogDirFailureChannel to asynchronously handle log dir failure * @param producerStateManager The ProducerStateManager instance (if any) containing state associated * with the existingSegments - * + * @param logPrefix The logging prefix * @throws IOException if the file can't be renamed and still exists */ private[log] def deleteSegmentFiles(segmentsToDelete: Iterable[LogSegment], @@ -2370,11 +2386,12 @@ object Log extends Logging { config: LogConfig, scheduler: Scheduler, logDirFailureChannel: LogDirFailureChannel, - producerStateManager: ProducerStateManager): Unit = { + producerStateManager: ProducerStateManager, + logPrefix: String): Unit = { segmentsToDelete.foreach(_.changeFileSuffixes("", Log.DeletedFileSuffix)) def deleteSegments(): Unit = { - info(s"Deleting segment files ${segmentsToDelete.mkString(",")}") + info(s"${logPrefix}Deleting segment files ${segmentsToDelete.mkString(",")}") val parentDir = dir.getParent maybeHandleIOException(logDirFailureChannel, parentDir, s"Error while deleting segments for $topicPartition in dir $parentDir") { segmentsToDelete.foreach { segment => @@ -2429,6 +2446,7 @@ object Log extends Logging { * @param time The time instance used for checking the clock * @param reloadFromCleanShutdown True if the producer state is being built after a clean shutdown, * false otherwise. + * @param logPrefix The logging prefix */ private[log] def rebuildProducerState(producerStateManager: ProducerStateManager, segments: LogSegments, @@ -2436,16 +2454,17 @@ object Log extends Logging { lastOffset: Long, recordVersion: RecordVersion, time: Time, - reloadFromCleanShutdown: Boolean): Unit = { - val allSegments = segments.values + reloadFromCleanShutdown: Boolean, + logPrefix: String): Unit = { val offsetsToSnapshot = - if (allSegments.nonEmpty) { - val nextLatestSegmentBaseOffset = segments.lowerSegment(allSegments.last.baseOffset).map(_.baseOffset) - Seq(nextLatestSegmentBaseOffset, Some(allSegments.last.baseOffset), Some(lastOffset)) + if (segments.nonEmpty) { + val lastSegmentBaseOffset = segments.lastSegment.get.baseOffset + val nextLatestSegmentBaseOffset = segments.lowerSegment(lastSegmentBaseOffset).map(_.baseOffset) + Seq(nextLatestSegmentBaseOffset, Some(lastSegmentBaseOffset), Some(lastOffset)) } else { Seq(Some(lastOffset)) } - info(s"Loading producer state till offset $lastOffset with message format version ${recordVersion.value}") + info(s"${logPrefix}Loading producer state till offset $lastOffset with message format version ${recordVersion.value}") // We want to avoid unnecessary scanning of the log to build the producer state when the broker is being // upgraded. The basic idea is to use the absence of producer snapshot files to detect the upgrade case, @@ -2469,7 +2488,7 @@ object Log extends Logging { producerStateManager.takeSnapshot() } } else { - info(s"Reloading from producer snapshot and rebuilding producer state from offset $lastOffset") + info(s"${logPrefix}Reloading from producer snapshot and rebuilding producer state from offset $lastOffset") val isEmptyBeforeTruncation = producerStateManager.isEmpty && producerStateManager.mapEndOffset >= lastOffset val producerStateLoadStart = time.milliseconds() producerStateManager.truncateAndReload(logStartOffset, lastOffset, time.milliseconds()) @@ -2508,7 +2527,7 @@ object Log extends Logging { } producerStateManager.updateMapEndOffset(lastOffset) producerStateManager.takeSnapshot() - info(s"Producer state recovery took ${segmentRecoveryStart - producerStateLoadStart}ms for snapshot load " + + info(s"${logPrefix}Producer state recovery took ${segmentRecoveryStart - producerStateLoadStart}ms for snapshot load " + s"and ${time.milliseconds() - segmentRecoveryStart}ms for segment recovery from offset $lastOffset") } } @@ -2535,6 +2554,7 @@ object Log extends Logging { * @param logDirFailureChannel The LogDirFailureChannel to asynchronously handle log dir failure * @param producerStateManager The ProducerStateManager instance (if any) containing state associated * with the existingSegments + * @param logPrefix The logging prefix * @return List of new segments that replace the input segment */ private[log] def splitOverflowedSegment(segment: LogSegment, @@ -2544,11 +2564,12 @@ object Log extends Logging { config: LogConfig, scheduler: Scheduler, logDirFailureChannel: LogDirFailureChannel, - producerStateManager: ProducerStateManager): List[LogSegment] = { + producerStateManager: ProducerStateManager, + logPrefix: String): List[LogSegment] = { require(Log.isLogFile(segment.log.file), s"Cannot split file ${segment.log.file.getAbsoluteFile}") require(segment.hasOverflow, "Split operation is only permitted for segments with overflow") - info(s"Splitting overflowed segment $segment") + info(s"${logPrefix}Splitting overflowed segment $segment") val newSegments = ListBuffer[LogSegment]() try { @@ -2581,9 +2602,9 @@ object Log extends Logging { s" before: ${segment.log.sizeInBytes} after: $totalSizeOfNewSegments") // replace old segment with new ones - info(s"Replacing overflowed segment $segment with split segments $newSegments") + info(s"${logPrefix}Replacing overflowed segment $segment with split segments $newSegments") replaceSegments(existingSegments, newSegments.toList, List(segment), isRecoveredSwapFile = false, - dir, topicPartition, config, scheduler, logDirFailureChannel, producerStateManager) + dir, topicPartition, config, scheduler, logDirFailureChannel, producerStateManager, logPrefix) newSegments.toList } catch { case e: Exception => @@ -2594,6 +2615,21 @@ object Log extends Logging { throw e } } + + /** + * Wraps the value of iterator.next() in an option. + * Note: this facility is a part of the Iterator class starting from scala v2.13. + * + * @param iterator + * @tparam T the type of object held within the iterator + * @return Some(iterator.next) if a next element exists, None otherwise. + */ + private def nextOption[T](iterator: Iterator[T]): Option[T] = { + if (iterator.hasNext) + Some(iterator.next()) + else + None + } } object LogMetricNames { diff --git a/core/src/main/scala/kafka/log/LogLoader.scala b/core/src/main/scala/kafka/log/LogLoader.scala index 6b28ec51bb91d..ef9ba000d6ffb 100644 --- a/core/src/main/scala/kafka/log/LogLoader.scala +++ b/core/src/main/scala/kafka/log/LogLoader.scala @@ -67,7 +67,7 @@ case class LoadLogParams(dir: File, maxProducerIdExpirationMs: Int, leaderEpochCache: Option[LeaderEpochFileCache], producerStateManager: ProducerStateManager) { - val logIdentifier: String = s"[LogLoader partition=$topicPartition, dir=${dir.getParent}]" + val logIdentifier: String = s"[LogLoader partition=$topicPartition, dir=${dir.getParent}] " } /** @@ -152,7 +152,8 @@ object LogLoader extends Logging { nextOffset, params.config.messageFormatVersion.recordVersion, params.time, - reloadFromCleanShutdown = params.hadCleanShutdown) + reloadFromCleanShutdown = params.hadCleanShutdown, + params.logIdentifier) val activeSegment = params.segments.lastSegment.get LoadedLogOffsets( @@ -172,7 +173,7 @@ object LogLoader extends Logging { private def removeTempFilesAndCollectSwapFiles(params: LoadLogParams): Set[File] = { def deleteIndicesIfExist(baseFile: File, suffix: String = ""): Unit = { - info(s"${params.logIdentifier} Deleting index files with suffix $suffix for baseFile $baseFile") + info(s"${params.logIdentifier}Deleting index files with suffix $suffix for baseFile $baseFile") val offset = offsetFromFile(baseFile) Files.deleteIfExists(Log.offsetIndexFile(params.dir, offset, suffix).toPath) Files.deleteIfExists(Log.timeIndexFile(params.dir, offset, suffix).toPath) @@ -188,7 +189,7 @@ object LogLoader extends Logging { throw new IOException(s"Could not read file $file") val filename = file.getName if (filename.endsWith(DeletedFileSuffix)) { - debug(s"${params.logIdentifier} Deleting stray temporary file ${file.getAbsolutePath}") + debug(s"${params.logIdentifier}Deleting stray temporary file ${file.getAbsolutePath}") Files.deleteIfExists(file.toPath) } else if (filename.endsWith(CleanedFileSuffix)) { minCleanedFileOffset = Math.min(offsetFromFileName(filename), minCleanedFileOffset) @@ -198,7 +199,7 @@ object LogLoader extends Logging { // if a log, delete the index files, complete the swap operation later // if an index just delete the index files, they will be rebuilt val baseFile = new File(CoreUtils.replaceSuffix(file.getPath, SwapFileSuffix, "")) - info(s"${params.logIdentifier} Found file ${file.getAbsolutePath} from interrupted swap operation.") + info(s"${params.logIdentifier}Found file ${file.getAbsolutePath} from interrupted swap operation.") if (Log.isIndexFile(baseFile)) { deleteIndicesIfExist(baseFile) } else if (Log.isLogFile(baseFile)) { @@ -213,7 +214,7 @@ object LogLoader extends Logging { // for more details about the split operation. val (invalidSwapFiles, validSwapFiles) = swapFiles.partition(file => offsetFromFile(file) >= minCleanedFileOffset) invalidSwapFiles.foreach { file => - debug(s"${params.logIdentifier} Deleting invalid swap file ${file.getAbsoluteFile} minCleanedFileOffset: $minCleanedFileOffset") + debug(s"${params.logIdentifier}Deleting invalid swap file ${file.getAbsoluteFile} minCleanedFileOffset: $minCleanedFileOffset") val baseFile = new File(CoreUtils.replaceSuffix(file.getPath, SwapFileSuffix, "")) deleteIndicesIfExist(baseFile, SwapFileSuffix) Files.deleteIfExists(file.toPath) @@ -221,7 +222,7 @@ object LogLoader extends Logging { // Now that we have deleted all .swap files that constitute an incomplete split operation, let's delete all .clean files cleanFiles.foreach { file => - debug(s"${params.logIdentifier} Deleting stray .clean file ${file.getAbsolutePath}") + debug(s"${params.logIdentifier}Deleting stray .clean file ${file.getAbsolutePath}") Files.deleteIfExists(file.toPath) } @@ -245,7 +246,7 @@ object LogLoader extends Logging { return fn } catch { case e: LogSegmentOffsetOverflowException => - info(s"${params.logIdentifier} Caught segment overflow error: ${e.getMessage}. Split segment and retry.") + info(s"${params.logIdentifier}Caught segment overflow error: ${e.getMessage}. Split segment and retry.") Log.splitOverflowedSegment( e.segment, params.segments, @@ -254,7 +255,8 @@ object LogLoader extends Logging { params.config, params.scheduler, params.logDirFailureChannel, - params.producerStateManager) + params.producerStateManager, + params.logIdentifier) } } throw new IllegalStateException() @@ -280,7 +282,7 @@ object LogLoader extends Logging { val offset = offsetFromFile(file) val logFile = Log.logFile(params.dir, offset) if (!logFile.exists) { - warn(s"${params.logIdentifier} Found an orphaned index file ${file.getAbsolutePath}, with no corresponding log file.") + warn(s"${params.logIdentifier}Found an orphaned index file ${file.getAbsolutePath}, with no corresponding log file.") Files.deleteIfExists(file.toPath) } } else if (isLogFile(file)) { @@ -297,11 +299,11 @@ object LogLoader extends Logging { try segment.sanityCheck(timeIndexFileNewlyCreated) catch { case _: NoSuchFileException => - error(s"${params.logIdentifier} Could not find offset index file corresponding to log file" + + error(s"${params.logIdentifier}Could not find offset index file corresponding to log file" + s" ${segment.log.file.getAbsolutePath}, recovering segment and rebuilding index files...") recoverSegment(segment, params) case e: CorruptIndexException => - warn(s"${params.logIdentifier} Found a corrupted index file corresponding to log file" + + warn(s"${params.logIdentifier}Found a corrupted index file corresponding to log file" + s" ${segment.log.file.getAbsolutePath} due to ${e.getMessage}}, recovering segment and" + " rebuilding index files...") recoverSegment(segment, params) @@ -322,7 +324,11 @@ object LogLoader extends Logging { * @throws LogSegmentOffsetOverflowException if the segment contains messages that cause index offset overflow */ private def recoverSegment(segment: LogSegment, params: LoadLogParams): Int = { - val producerStateManager = new ProducerStateManager(params.topicPartition, params.dir, params.maxProducerIdExpirationMs) + val producerStateManager = new ProducerStateManager( + params.topicPartition, + params.dir, + params.maxProducerIdExpirationMs, + params.time) Log.rebuildProducerState( producerStateManager, params.segments, @@ -330,7 +336,8 @@ object LogLoader extends Logging { segment.baseOffset, params.config.messageFormatVersion.recordVersion, params.time, - reloadFromCleanShutdown = false) + reloadFromCleanShutdown = false, + params.logIdentifier) val bytesTruncated = segment.recover(producerStateManager, params.leaderEpochCache) // once we have recovered the segment's data, take a snapshot to ensure that we won't // need to reload the same segment again while recovering another segment. @@ -390,7 +397,8 @@ object LogLoader extends Logging { params.config, params.scheduler, params.logDirFailureChannel, - params.producerStateManager) + params.producerStateManager, + params.logIdentifier) } } @@ -434,20 +442,20 @@ object LogLoader extends Logging { while (unflushed.hasNext && !truncated) { val segment = unflushed.next() - info(s"${params.logIdentifier} Recovering unflushed segment ${segment.baseOffset}") + info(s"${params.logIdentifier}Recovering unflushed segment ${segment.baseOffset}") val truncatedBytes = try { recoverSegment(segment, params) } catch { case _: InvalidOffsetException => val startOffset = segment.baseOffset - warn(s"${params.logIdentifier} Found invalid offset during recovery. Deleting the" + + warn(s"${params.logIdentifier}Found invalid offset during recovery. Deleting the" + s" corrupt segment and creating an empty one with starting offset $startOffset") segment.truncateTo(startOffset) } if (truncatedBytes > 0) { // we had an invalid message, delete all remaining log - warn(s"${params.logIdentifier} Corruption found in segment ${segment.baseOffset}," + + warn(s"${params.logIdentifier}Corruption found in segment ${segment.baseOffset}," + s" truncating to offset ${segment.readNextOffset}") removeAndDeleteSegmentsAsync(unflushed.toList, params) truncated = true @@ -519,7 +527,8 @@ object LogLoader extends Logging { params.config, params.scheduler, params.logDirFailureChannel, - params.producerStateManager) + params.producerStateManager, + params.logIdentifier) } } } diff --git a/core/src/main/scala/kafka/log/LogSegments.scala b/core/src/main/scala/kafka/log/LogSegments.scala index d9e564ed4b13d..d6886d7fdef13 100644 --- a/core/src/main/scala/kafka/log/LogSegments.scala +++ b/core/src/main/scala/kafka/log/LogSegments.scala @@ -17,7 +17,6 @@ package kafka.log import java.io.File -import java.lang.{Long => JLong} import java.util.Map import java.util.concurrent.{ConcurrentNavigableMap, ConcurrentSkipListMap} @@ -36,7 +35,7 @@ import scala.jdk.CollectionConverters._ class LogSegments(topicPartition: TopicPartition) { /* the segments of the log with key being LogSegment base offset and value being a LogSegment */ - private val segments: ConcurrentNavigableMap[java.lang.Long, LogSegment] = new ConcurrentSkipListMap[java.lang.Long, LogSegment] + private val segments: ConcurrentNavigableMap[Long, LogSegment] = new ConcurrentSkipListMap[Long, LogSegment] /** * @return true if the segments are empty, false otherwise. @@ -157,7 +156,7 @@ class LogSegments(topicPartition: TopicPartition) { * if it exists. */ @threadsafe - def floorEntry(offset: Long): Option[Map.Entry[JLong, LogSegment]] = Option(segments.floorEntry(offset)) + private def floorEntry(offset: Long): Option[Map.Entry[Long, LogSegment]] = Option(segments.floorEntry(offset)) /** * @return the log segment with the greatest offset less than or equal to the given offset, @@ -171,7 +170,7 @@ class LogSegments(topicPartition: TopicPartition) { * if it exists. */ @threadsafe - def lowerEntry(offset: Long): Option[Map.Entry[JLong, LogSegment]] = Option(segments.lowerEntry(offset)) + private def lowerEntry(offset: Long): Option[Map.Entry[Long, LogSegment]] = Option(segments.lowerEntry(offset)) /** * @return the log segment with the greatest offset strictly less than the given offset, @@ -185,7 +184,7 @@ class LogSegments(topicPartition: TopicPartition) { * if it exists. */ @threadsafe - def higherEntry(offset: Long): Option[Map.Entry[JLong, LogSegment]] = Option(segments.higherEntry(offset)) + def higherEntry(offset: Long): Option[Map.Entry[Long, LogSegment]] = Option(segments.higherEntry(offset)) /** * @return the log segment with the smallest offset strictly greater than the given offset, @@ -198,7 +197,7 @@ class LogSegments(topicPartition: TopicPartition) { * @return the entry associated with the smallest offset, if it exists. */ @threadsafe - def firstEntry: Option[Map.Entry[JLong, LogSegment]] = Option(segments.firstEntry) + def firstEntry: Option[Map.Entry[Long, LogSegment]] = Option(segments.firstEntry) /** * @return the log segment associated with the smallest offset, if it exists. @@ -210,11 +209,23 @@ class LogSegments(topicPartition: TopicPartition) { * @return the entry associated with the greatest offset, if it exists. */ @threadsafe - def lastEntry: Option[Map.Entry[JLong, LogSegment]] = Option(segments.lastEntry) + def lastEntry: Option[Map.Entry[Long, LogSegment]] = Option(segments.lastEntry) /** * @return the log segment with the greatest offset, if it exists. */ @threadsafe def lastSegment: Option[LogSegment] = lastEntry.map(_.getValue) + + /** + * @return an iterable with log segments ordered from lowest base offset to highest, + * each segment returned has a base offset strictly greater than the provided baseOffset. + */ + def higherSegments(baseOffset: Long): Iterable[LogSegment] = { + val view = + Option(segments.higherKey(baseOffset)).map { + higherOffset => segments.tailMap(higherOffset, true) + }.getOrElse(collection.immutable.Map[Long, LogSegment]().asJava) + view.values.asScala + } } diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala index 056be10be181c..925c60294a6af 100644 --- a/core/src/main/scala/kafka/log/LogValidator.scala +++ b/core/src/main/scala/kafka/log/LogValidator.scala @@ -20,7 +20,7 @@ import java.nio.ByteBuffer import kafka.api.{ApiVersion, KAFKA_2_1_IV0} import kafka.common.{LongRef, RecordValidationException} import kafka.message.{CompressionCodec, NoCompressionCodec, ZStdCompressionCodec} -import kafka.server.BrokerTopicStats +import kafka.server.{BrokerTopicStats, RequestLocal} import kafka.utils.Logging import org.apache.kafka.common.errors.{CorruptRecordException, InvalidTimestampException, UnsupportedCompressionTypeException, UnsupportedForMessageFormatException} import org.apache.kafka.common.record.{AbstractRecords, CompressionType, MemoryRecords, Record, RecordBatch, RecordConversionStats, TimestampType} @@ -28,7 +28,7 @@ import org.apache.kafka.common.InvalidRecordException import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests.ProduceResponse.RecordError -import org.apache.kafka.common.utils.{BufferSupplier, Time} +import org.apache.kafka.common.utils.Time import scala.collection.{Seq, mutable} import scala.jdk.CollectionConverters._ @@ -95,7 +95,8 @@ private[log] object LogValidator extends Logging { partitionLeaderEpoch: Int, origin: AppendOrigin, interBrokerProtocolVersion: ApiVersion, - brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = { + brokerTopicStats: BrokerTopicStats, + requestLocal: RequestLocal): ValidationAndOffsetAssignResult = { if (sourceCodec == NoCompressionCodec && targetCodec == NoCompressionCodec) { // check the magic value if (!records.hasMatchingMagic(magic)) @@ -106,8 +107,9 @@ private[log] object LogValidator extends Logging { assignOffsetsNonCompressed(records, topicPartition, offsetCounter, now, compactedTopic, timestampType, timestampDiffMaxMs, partitionLeaderEpoch, origin, magic, brokerTopicStats) } else { - validateMessagesAndAssignOffsetsCompressed(records, topicPartition, offsetCounter, time, now, sourceCodec, targetCodec, compactedTopic, - magic, timestampType, timestampDiffMaxMs, partitionLeaderEpoch, origin, interBrokerProtocolVersion, brokerTopicStats) + validateMessagesAndAssignOffsetsCompressed(records, topicPartition, offsetCounter, time, now, sourceCodec, + targetCodec, compactedTopic, magic, timestampType, timestampDiffMaxMs, partitionLeaderEpoch, origin, + interBrokerProtocolVersion, brokerTopicStats, requestLocal) } } @@ -232,6 +234,8 @@ private[log] object LogValidator extends Logging { (first.producerId, first.producerEpoch, first.baseSequence, first.isTransactional) } + // The current implementation of BufferSupplier is naive and works best when the buffer size + // cardinality is low, so don't use it here val newBuffer = ByteBuffer.allocate(sizeInBytesAfterConversion) val builder = MemoryRecords.builder(newBuffer, toMagicValue, CompressionType.NONE, timestampType, offsetCounter.value, now, producerId, producerEpoch, sequence, isTransactional, partitionLeaderEpoch) @@ -290,7 +294,9 @@ private[log] object LogValidator extends Logging { var offsetOfMaxBatchTimestamp = -1L val recordErrors = new ArrayBuffer[ApiRecordError](0) - // this is a hot path and we want to avoid any unnecessary allocations. + // This is a hot path and we want to avoid any unnecessary allocations. + // That said, there is no benefit in using `skipKeyValueIterator` for the uncompressed + // case since we don't do key/value copies in this path (we just slice the ByteBuffer) var batchIndex = 0 batch.forEach { record => validateRecord(batch, topicPartition, record, batchIndex, now, timestampType, @@ -360,7 +366,8 @@ private[log] object LogValidator extends Logging { partitionLeaderEpoch: Int, origin: AppendOrigin, interBrokerProtocolVersion: ApiVersion, - brokerTopicStats: BrokerTopicStats): ValidationAndOffsetAssignResult = { + brokerTopicStats: BrokerTopicStats, + requestLocal: RequestLocal): ValidationAndOffsetAssignResult = { if (targetCodec == ZStdCompressionCodec && interBrokerProtocolVersion < KAFKA_2_1_IV0) throw new UnsupportedCompressionTypeException("Produce requests to inter.broker.protocol.version < 2.1 broker " + @@ -404,9 +411,9 @@ private[log] object LogValidator extends Logging { // if we are on version 2 and beyond, and we know we are going for in place assignment, // then we can optimize the iterator to skip key / value / headers since they would not be used at all val recordsIterator = if (inPlaceAssignment && firstBatch.magic >= RecordBatch.MAGIC_VALUE_V2) - batch.skipKeyValueIterator(BufferSupplier.NO_CACHING) + batch.skipKeyValueIterator(requestLocal.bufferSupplier) else - batch.streamingIterator(BufferSupplier.NO_CACHING) + batch.streamingIterator(requestLocal.bufferSupplier) try { val recordErrors = new ArrayBuffer[ApiRecordError](0) @@ -499,6 +506,8 @@ private[log] object LogValidator extends Logging { val startNanos = time.nanoseconds val estimatedSize = AbstractRecords.estimateSizeInBytes(magic, offsetCounter.value, compressionType, validatedRecords.asJava) + // The current implementation of BufferSupplier is naive and works best when the buffer size + // cardinality is low, so don't use it here val buffer = ByteBuffer.allocate(estimatedSize) val builder = MemoryRecords.builder(buffer, magic, compressionType, timestampType, offsetCounter.value, logAppendTime, producerId, producerEpoch, baseSequence, isTransactional, partitionLeaderEpoch) diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index ca1982bba276d..b4e7ac1597201 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -21,7 +21,6 @@ import java.nio.ByteBuffer import java.nio.channels.FileChannel import java.nio.file.{Files, StandardOpenOption} import java.util.concurrent.ConcurrentSkipListMap - import kafka.log.Log.offsetFromFile import kafka.server.LogOffsetMetadata import kafka.utils.{Logging, nonthreadsafe, threadsafe} @@ -29,6 +28,7 @@ import org.apache.kafka.common.{KafkaException, TopicPartition} import org.apache.kafka.common.errors._ import org.apache.kafka.common.protocol.types._ import org.apache.kafka.common.record.{ControlRecordType, DefaultRecordBatch, EndTransactionMarker, RecordBatch} +import org.apache.kafka.common.utils.Time import org.apache.kafka.common.utils.{ByteUtils, Crc32C} import scala.jdk.CollectionConverters._ @@ -484,7 +484,8 @@ object ProducerStateManager { @nonthreadsafe class ProducerStateManager(val topicPartition: TopicPartition, @volatile var _logDir: File, - val maxProducerIdExpirationMs: Int = 60 * 60 * 1000) extends Logging { + val maxProducerIdExpirationMs: Int = 60 * 60 * 1000, + val time: Time = Time.SYSTEM) extends Logging { import ProducerStateManager._ import java.util @@ -718,8 +719,10 @@ class ProducerStateManager(val topicPartition: TopicPartition, // If not a new offset, then it is not worth taking another snapshot if (lastMapOffset > lastSnapOffset) { val snapshotFile = SnapshotFile(Log.producerSnapshotFile(_logDir, lastMapOffset)) - info(s"Writing producer snapshot at offset $lastMapOffset") + val start = time.hiResClockMs() writeSnapshot(snapshotFile.file, producers) + info(s"Wrote producer snapshot at offset $lastMapOffset with ${producers.size} producer ids in ${time.hiResClockMs() - start} ms.") + snapshots.put(snapshotFile.offset, snapshotFile) // Update the last snap offset according to the serialized map @@ -730,7 +733,7 @@ class ProducerStateManager(val topicPartition: TopicPartition, /** * Update the parentDir for this ProducerStateManager and all of the snapshot files which it manages. */ - def updateParentDir(parentDir: File): Unit ={ + def updateParentDir(parentDir: File): Unit = { _logDir = parentDir snapshots.forEach((_, s) => s.updateParentDir(parentDir)) } diff --git a/core/src/main/scala/kafka/raft/KafkaMetadataLog.scala b/core/src/main/scala/kafka/raft/KafkaMetadataLog.scala index 6ac39b7f6cbb5..7b5f83dd53ac7 100644 --- a/core/src/main/scala/kafka/raft/KafkaMetadataLog.scala +++ b/core/src/main/scala/kafka/raft/KafkaMetadataLog.scala @@ -19,10 +19,9 @@ package kafka.raft import java.io.File import java.nio.file.{Files, NoSuchFileException, Path} import java.util.{Optional, Properties} - import kafka.api.ApiVersion import kafka.log.{AppendOrigin, Log, LogConfig, LogOffsetSnapshot, SnapshotGenerated} -import kafka.server.{BrokerTopicStats, FetchHighWatermark, FetchLogEnd, LogDirFailureChannel} +import kafka.server.{BrokerTopicStats, FetchHighWatermark, FetchLogEnd, LogDirFailureChannel, RequestLocal} import kafka.utils.{CoreUtils, Logging, Scheduler} import org.apache.kafka.common.record.{MemoryRecords, Records} import org.apache.kafka.common.utils.Time @@ -76,7 +75,8 @@ final class KafkaMetadataLog private ( handleAndConvertLogAppendInfo( log.appendAsLeader(records.asInstanceOf[MemoryRecords], leaderEpoch = epoch, - origin = AppendOrigin.RaftLeader + origin = AppendOrigin.RaftLeader, + requestLocal = RequestLocal.NoCaching ) ) } diff --git a/core/src/main/scala/kafka/server/ControllerApis.scala b/core/src/main/scala/kafka/server/ControllerApis.scala index 47bc19d69553a..fa0dc79465af3 100644 --- a/core/src/main/scala/kafka/server/ControllerApis.scala +++ b/core/src/main/scala/kafka/server/ControllerApis.scala @@ -76,7 +76,7 @@ class ControllerApis(val requestChannel: RequestChannel, val requestHelper = new RequestHandlerHelper(requestChannel, quotas, time) private val aclApis = new AclApis(authHelper, authorizer, requestHelper, "controller", config) - override def handle(request: RequestChannel.Request): Unit = { + override def handle(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { try { request.header.apiKey match { case ApiKeys.FETCH => handleFetch(request) @@ -97,7 +97,7 @@ class ControllerApis(val requestChannel: RequestChannel, case ApiKeys.INCREMENTAL_ALTER_CONFIGS => handleIncrementalAlterConfigs(request) case ApiKeys.ALTER_PARTITION_REASSIGNMENTS => handleAlterPartitionReassignments(request) case ApiKeys.LIST_PARTITION_REASSIGNMENTS => handleListPartitionReassignments(request) - case ApiKeys.ENVELOPE => handleEnvelopeRequest(request) + case ApiKeys.ENVELOPE => handleEnvelopeRequest(request, requestLocal) case ApiKeys.SASL_HANDSHAKE => handleSaslHandshakeRequest(request) case ApiKeys.SASL_AUTHENTICATE => handleSaslAuthenticateRequest(request) case ApiKeys.CREATE_PARTITIONS => handleCreatePartitions(request) @@ -113,12 +113,12 @@ class ControllerApis(val requestChannel: RequestChannel, } } - def handleEnvelopeRequest(request: RequestChannel.Request): Unit = { + def handleEnvelopeRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { if (!authHelper.authorize(request.context, CLUSTER_ACTION, CLUSTER, CLUSTER_NAME)) { requestHelper.sendErrorResponseMaybeThrottle(request, new ClusterAuthorizationException( s"Principal ${request.context.principal} does not have required CLUSTER_ACTION for envelope")) } else { - EnvelopeUtils.handleEnvelopeRequest(request, requestChannel.metrics, handle) + EnvelopeUtils.handleEnvelopeRequest(request, requestChannel.metrics, handle(_, requestLocal)) } } diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index d75e4ae3651d7..c5e140494b31e 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -144,7 +144,7 @@ class KafkaApis(val requestChannel: RequestChannel, /** * Top-level method that handles all requests and multiplexes to the right api */ - override def handle(request: RequestChannel.Request): Unit = { + override def handle(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { try { trace(s"Handling request:${request.requestDesc(true)} from connection ${request.context.connectionId};" + s"securityProtocol:${request.context.securityProtocol},principal:${request.context.principal}") @@ -156,21 +156,21 @@ class KafkaApis(val requestChannel: RequestChannel, } request.header.apiKey match { - case ApiKeys.PRODUCE => handleProduceRequest(request) + case ApiKeys.PRODUCE => handleProduceRequest(request, requestLocal) case ApiKeys.FETCH => handleFetchRequest(request) case ApiKeys.LIST_OFFSETS => handleListOffsetRequest(request) case ApiKeys.METADATA => handleTopicMetadataRequest(request) case ApiKeys.LEADER_AND_ISR => handleLeaderAndIsrRequest(request) case ApiKeys.STOP_REPLICA => handleStopReplicaRequest(request) - case ApiKeys.UPDATE_METADATA => handleUpdateMetadataRequest(request) + case ApiKeys.UPDATE_METADATA => handleUpdateMetadataRequest(request, requestLocal) case ApiKeys.CONTROLLED_SHUTDOWN => handleControlledShutdownRequest(request) - case ApiKeys.OFFSET_COMMIT => handleOffsetCommitRequest(request) + case ApiKeys.OFFSET_COMMIT => handleOffsetCommitRequest(request, requestLocal) case ApiKeys.OFFSET_FETCH => handleOffsetFetchRequest(request) case ApiKeys.FIND_COORDINATOR => handleFindCoordinatorRequest(request) - case ApiKeys.JOIN_GROUP => handleJoinGroupRequest(request) + case ApiKeys.JOIN_GROUP => handleJoinGroupRequest(request, requestLocal) case ApiKeys.HEARTBEAT => handleHeartbeatRequest(request) case ApiKeys.LEAVE_GROUP => handleLeaveGroupRequest(request) - case ApiKeys.SYNC_GROUP => handleSyncGroupRequest(request) + case ApiKeys.SYNC_GROUP => handleSyncGroupRequest(request, requestLocal) case ApiKeys.DESCRIBE_GROUPS => handleDescribeGroupRequest(request) case ApiKeys.LIST_GROUPS => handleListGroupsRequest(request) case ApiKeys.SASL_HANDSHAKE => handleSaslHandshakeRequest(request) @@ -178,13 +178,13 @@ class KafkaApis(val requestChannel: RequestChannel, case ApiKeys.CREATE_TOPICS => maybeForwardToController(request, handleCreateTopicsRequest) case ApiKeys.DELETE_TOPICS => maybeForwardToController(request, handleDeleteTopicsRequest) case ApiKeys.DELETE_RECORDS => handleDeleteRecordsRequest(request) - case ApiKeys.INIT_PRODUCER_ID => handleInitProducerIdRequest(request) + case ApiKeys.INIT_PRODUCER_ID => handleInitProducerIdRequest(request, requestLocal) case ApiKeys.OFFSET_FOR_LEADER_EPOCH => handleOffsetForLeaderEpochRequest(request) - case ApiKeys.ADD_PARTITIONS_TO_TXN => handleAddPartitionToTxnRequest(request) - case ApiKeys.ADD_OFFSETS_TO_TXN => handleAddOffsetsToTxnRequest(request) - case ApiKeys.END_TXN => handleEndTxnRequest(request) - case ApiKeys.WRITE_TXN_MARKERS => handleWriteTxnMarkersRequest(request) - case ApiKeys.TXN_OFFSET_COMMIT => handleTxnOffsetCommitRequest(request) + case ApiKeys.ADD_PARTITIONS_TO_TXN => handleAddPartitionToTxnRequest(request, requestLocal) + case ApiKeys.ADD_OFFSETS_TO_TXN => handleAddOffsetsToTxnRequest(request, requestLocal) + case ApiKeys.END_TXN => handleEndTxnRequest(request, requestLocal) + case ApiKeys.WRITE_TXN_MARKERS => handleWriteTxnMarkersRequest(request, requestLocal) + case ApiKeys.TXN_OFFSET_COMMIT => handleTxnOffsetCommitRequest(request, requestLocal) case ApiKeys.DESCRIBE_ACLS => handleDescribeAcls(request) case ApiKeys.CREATE_ACLS => maybeForwardToController(request, handleCreateAcls) case ApiKeys.DELETE_ACLS => maybeForwardToController(request, handleDeleteAcls) @@ -198,19 +198,19 @@ class KafkaApis(val requestChannel: RequestChannel, case ApiKeys.RENEW_DELEGATION_TOKEN => maybeForwardToController(request, handleRenewTokenRequest) case ApiKeys.EXPIRE_DELEGATION_TOKEN => maybeForwardToController(request, handleExpireTokenRequest) case ApiKeys.DESCRIBE_DELEGATION_TOKEN => handleDescribeTokensRequest(request) - case ApiKeys.DELETE_GROUPS => handleDeleteGroupsRequest(request) + case ApiKeys.DELETE_GROUPS => handleDeleteGroupsRequest(request, requestLocal) case ApiKeys.ELECT_LEADERS => handleElectReplicaLeader(request) case ApiKeys.INCREMENTAL_ALTER_CONFIGS => maybeForwardToController(request, handleIncrementalAlterConfigsRequest) case ApiKeys.ALTER_PARTITION_REASSIGNMENTS => maybeForwardToController(request, handleAlterPartitionReassignmentsRequest) case ApiKeys.LIST_PARTITION_REASSIGNMENTS => handleListPartitionReassignmentsRequest(request) - case ApiKeys.OFFSET_DELETE => handleOffsetDeleteRequest(request) + case ApiKeys.OFFSET_DELETE => handleOffsetDeleteRequest(request, requestLocal) case ApiKeys.DESCRIBE_CLIENT_QUOTAS => handleDescribeClientQuotasRequest(request) case ApiKeys.ALTER_CLIENT_QUOTAS => maybeForwardToController(request, handleAlterClientQuotasRequest) case ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS => handleDescribeUserScramCredentialsRequest(request) case ApiKeys.ALTER_USER_SCRAM_CREDENTIALS => maybeForwardToController(request, handleAlterUserScramCredentialsRequest) case ApiKeys.ALTER_ISR => handleAlterIsrRequest(request) case ApiKeys.UPDATE_FEATURES => maybeForwardToController(request, handleUpdateFeatures) - case ApiKeys.ENVELOPE => handleEnvelope(request) + case ApiKeys.ENVELOPE => handleEnvelope(request, requestLocal) case ApiKeys.DESCRIBE_CLUSTER => handleDescribeCluster(request) case ApiKeys.DESCRIBE_PRODUCERS => handleDescribeProducersRequest(request) case ApiKeys.UNREGISTER_BROKER => maybeForwardToController(request, handleUnregisterBrokerRequest) @@ -314,7 +314,7 @@ class KafkaApis(val requestChannel: RequestChannel, CoreUtils.swallow(replicaManager.replicaFetcherManager.shutdownIdleFetcherThreads(), this) } - def handleUpdateMetadataRequest(request: RequestChannel.Request): Unit = { + def handleUpdateMetadataRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { val zkSupport = metadataSupport.requireZkOrThrow(KafkaApis.shouldNeverReceive(request)) val correlationId = request.header.correlationId val updateMetadataRequest = request.body[UpdateMetadataRequest] @@ -330,13 +330,14 @@ class KafkaApis(val requestChannel: RequestChannel, } else { val deletedPartitions = replicaManager.maybeUpdateMetadataCache(correlationId, updateMetadataRequest) if (deletedPartitions.nonEmpty) - groupCoordinator.handleDeletedPartitions(deletedPartitions) + groupCoordinator.handleDeletedPartitions(deletedPartitions, requestLocal) if (zkSupport.adminManager.hasDelayedTopicOperations) { updateMetadataRequest.partitionStates.forEach { partitionState => zkSupport.adminManager.tryCompleteDelayedTopicOperations(partitionState.topicName) } } + quotas.clientQuotaCallback.foreach { callback => if (callback.updateClusterMetadata(metadataCache.getClusterMetadata(clusterId, request.context.listenerName))) { quotas.fetch.updateQuotaMetricConfigs() @@ -380,7 +381,7 @@ class KafkaApis(val requestChannel: RequestChannel, /** * Handle an offset commit request */ - def handleOffsetCommitRequest(request: RequestChannel.Request): Unit = { + def handleOffsetCommitRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { val header = request.header val offsetCommitRequest = request.body[OffsetCommitRequest] @@ -509,7 +510,8 @@ class KafkaApis(val requestChannel: RequestChannel, Option(offsetCommitRequest.data.groupInstanceId), offsetCommitRequest.data.generationId, partitionData, - sendResponseCallback) + sendResponseCallback, + requestLocal) } } } @@ -517,7 +519,7 @@ class KafkaApis(val requestChannel: RequestChannel, /** * Handle a produce request */ - def handleProduceRequest(request: RequestChannel.Request): Unit = { + def handleProduceRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { val produceRequest = request.body[ProduceRequest] val requestSize = request.sizeInBytes @@ -639,6 +641,7 @@ class KafkaApis(val requestChannel: RequestChannel, internalTopicsAllowed = internalTopicsAllowed, origin = AppendOrigin.Client, entriesPerPartition = authorizedRequestInfo, + requestLocal = requestLocal, responseCallback = sendResponseCallback, recordConversionStatsCallback = processingStatsCallback) @@ -1448,7 +1451,7 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleJoinGroupRequest(request: RequestChannel.Request): Unit = { + def handleJoinGroupRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { val joinGroupRequest = request.body[JoinGroupRequest] // the callback for sending a join-group response @@ -1507,11 +1510,12 @@ class KafkaApis(val requestChannel: RequestChannel, joinGroupRequest.data.sessionTimeoutMs, joinGroupRequest.data.protocolType, protocols, - sendResponseCallback) + sendResponseCallback, + requestLocal) } } - def handleSyncGroupRequest(request: RequestChannel.Request): Unit = { + def handleSyncGroupRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { val syncGroupRequest = request.body[SyncGroupRequest] def sendResponseCallback(syncGroupResult: SyncGroupResult): Unit = { @@ -1550,19 +1554,20 @@ class KafkaApis(val requestChannel: RequestChannel, Option(syncGroupRequest.data.protocolName), Option(syncGroupRequest.data.groupInstanceId), assignmentMap.result(), - sendResponseCallback + sendResponseCallback, + requestLocal ) } } - def handleDeleteGroupsRequest(request: RequestChannel.Request): Unit = { + def handleDeleteGroupsRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { val deleteGroupsRequest = request.body[DeleteGroupsRequest] val groups = deleteGroupsRequest.data.groupsNames.asScala.distinct val (authorizedGroups, unauthorizedGroups) = authHelper.partitionSeqByAuthorized(request.context, DELETE, GROUP, groups)(identity) - val groupDeletionResult = groupCoordinator.handleDeleteGroups(authorizedGroups.toSet) ++ + val groupDeletionResult = groupCoordinator.handleDeleteGroups(authorizedGroups.toSet, requestLocal) ++ unauthorizedGroups.map(_ -> Errors.GROUP_AUTHORIZATION_FAILED) requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => { @@ -1990,7 +1995,7 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleInitProducerIdRequest(request: RequestChannel.Request): Unit = { + def handleInitProducerIdRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { val initProducerIdRequest = request.body[InitProducerIdRequest] val transactionalId = initProducerIdRequest.data.transactionalId @@ -2035,12 +2040,12 @@ class KafkaApis(val requestChannel: RequestChannel, producerIdAndEpoch match { case Right(producerIdAndEpoch) => txnCoordinator.handleInitProducerId(transactionalId, initProducerIdRequest.data.transactionTimeoutMs, - producerIdAndEpoch, sendResponseCallback) + producerIdAndEpoch, sendResponseCallback, requestLocal) case Left(error) => requestHelper.sendErrorResponseMaybeThrottle(request, error.exception) } } - def handleEndTxnRequest(request: RequestChannel.Request): Unit = { + def handleEndTxnRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { ensureInterBrokerVersion(KAFKA_0_11_0_IV0) val endTxnRequest = request.body[EndTxnRequest] val transactionalId = endTxnRequest.data.transactionalId @@ -2071,7 +2076,8 @@ class KafkaApis(val requestChannel: RequestChannel, endTxnRequest.data.producerId, endTxnRequest.data.producerEpoch, endTxnRequest.result(), - sendResponseCallback) + sendResponseCallback, + requestLocal) } else requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => new EndTxnResponse(new EndTxnResponseData() @@ -2080,7 +2086,7 @@ class KafkaApis(val requestChannel: RequestChannel, ) } - def handleWriteTxnMarkersRequest(request: RequestChannel.Request): Unit = { + def handleWriteTxnMarkersRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { ensureInterBrokerVersion(KAFKA_0_11_0_IV0) authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) val writeTxnMarkersRequest = request.body[WriteTxnMarkersRequest] @@ -2175,6 +2181,7 @@ class KafkaApis(val requestChannel: RequestChannel, internalTopicsAllowed = true, origin = AppendOrigin.Coordinator, entriesPerPartition = controlRecords, + requestLocal = requestLocal, responseCallback = maybeSendResponseCallback(producerId, marker.transactionResult)) } } @@ -2190,7 +2197,7 @@ class KafkaApis(val requestChannel: RequestChannel, throw new UnsupportedVersionException(s"inter.broker.protocol.version: ${config.interBrokerProtocolVersion.version} is less than the required version: ${version.version}") } - def handleAddPartitionToTxnRequest(request: RequestChannel.Request): Unit = { + def handleAddPartitionToTxnRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { ensureInterBrokerVersion(KAFKA_0_11_0_IV0) val addPartitionsToTxnRequest = request.body[AddPartitionsToTxnRequest] val transactionalId = addPartitionsToTxnRequest.data.transactionalId @@ -2240,7 +2247,6 @@ class KafkaApis(val requestChannel: RequestChannel, responseBody } - requestHelper.sendResponseMaybeThrottle(request, createResponse) } @@ -2248,12 +2254,13 @@ class KafkaApis(val requestChannel: RequestChannel, addPartitionsToTxnRequest.data.producerId, addPartitionsToTxnRequest.data.producerEpoch, authorizedPartitions, - sendResponseCallback) + sendResponseCallback, + requestLocal) } } } - def handleAddOffsetsToTxnRequest(request: RequestChannel.Request): Unit = { + def handleAddOffsetsToTxnRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { ensureInterBrokerVersion(KAFKA_0_11_0_IV0) val addOffsetsToTxnRequest = request.body[AddOffsetsToTxnRequest] val transactionalId = addOffsetsToTxnRequest.data.transactionalId @@ -2298,11 +2305,12 @@ class KafkaApis(val requestChannel: RequestChannel, addOffsetsToTxnRequest.data.producerId, addOffsetsToTxnRequest.data.producerEpoch, Set(offsetTopicPartition), - sendResponseCallback) + sendResponseCallback, + requestLocal) } } - def handleTxnOffsetCommitRequest(request: RequestChannel.Request): Unit = { + def handleTxnOffsetCommitRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { ensureInterBrokerVersion(KAFKA_0_11_0_IV0) val header = request.header val txnOffsetCommitRequest = request.body[TxnOffsetCommitRequest] @@ -2367,7 +2375,8 @@ class KafkaApis(val requestChannel: RequestChannel, Option(txnOffsetCommitRequest.data.groupInstanceId), txnOffsetCommitRequest.data.generationId, offsetMetadata, - sendResponseCallback) + sendResponseCallback, + requestLocal) } } } @@ -2877,7 +2886,7 @@ class KafkaApis(val requestChannel: RequestChannel, } } - def handleOffsetDeleteRequest(request: RequestChannel.Request): Unit = { + def handleOffsetDeleteRequest(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { val offsetDeleteRequest = request.body[OffsetDeleteRequest] val groupId = offsetDeleteRequest.data.groupId @@ -2901,7 +2910,7 @@ class KafkaApis(val requestChannel: RequestChannel, } val (groupError, authorizedTopicPartitionsErrors) = groupCoordinator.handleDeleteOffsets( - groupId, topicPartitions) + groupId, topicPartitions, requestLocal) topicPartitionErrors ++= authorizedTopicPartitionsErrors @@ -3129,7 +3138,7 @@ class KafkaApis(val requestChannel: RequestChannel, }) } - def handleEnvelope(request: RequestChannel.Request): Unit = { + def handleEnvelope(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { val zkSupport = metadataSupport.requireZkOrThrow(KafkaApis.shouldNeverReceive(request)) // If forwarding is not yet enabled or this request has been received on an invalid endpoint, @@ -3154,7 +3163,8 @@ class KafkaApis(val requestChannel: RequestChannel, s"Broker $brokerId is not the active controller")) return } - EnvelopeUtils.handleEnvelopeRequest(request, requestChannel.metrics, handle) + + EnvelopeUtils.handleEnvelopeRequest(request, requestChannel.metrics, handle(_, requestLocal)) } def handleDescribeProducersRequest(request: RequestChannel.Request): Unit = { diff --git a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala index f6d868ade7546..4d38c6efc3fd6 100755 --- a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala +++ b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala @@ -20,9 +20,9 @@ package kafka.server import kafka.network._ import kafka.utils._ import kafka.metrics.KafkaMetricsGroup + import java.util.concurrent.{CountDownLatch, TimeUnit} import java.util.concurrent.atomic.AtomicInteger - import com.yammer.metrics.core.Meter import org.apache.kafka.common.internals.FatalExitError import org.apache.kafka.common.utils.{KafkaThread, Time} @@ -31,7 +31,7 @@ import scala.collection.mutable import scala.jdk.CollectionConverters._ trait ApiRequestHandler { - def handle(request: RequestChannel.Request): Unit + def handle(request: RequestChannel.Request, requestLocal: RequestLocal): Unit } /** @@ -44,8 +44,9 @@ class KafkaRequestHandler(id: Int, val requestChannel: RequestChannel, apis: ApiRequestHandler, time: Time) extends Runnable with Logging { - this.logIdent = "[Kafka Request Handler " + id + " on Broker " + brokerId + "], " + this.logIdent = s"[Kafka Request Handler $id on Broker $brokerId], " private val shutdownComplete = new CountDownLatch(1) + private val requestLocal = RequestLocal.withThreadConfinedCaching @volatile private var stopped = false def run(): Unit = { @@ -64,17 +65,17 @@ class KafkaRequestHandler(id: Int, req match { case RequestChannel.ShutdownRequest => debug(s"Kafka request handler $id on broker $brokerId received shut down command") - shutdownComplete.countDown() + completeShutdown() return case request: RequestChannel.Request => try { request.requestDequeueTimeNanos = endTime trace(s"Kafka request handler $id on broker $brokerId handling request $request") - apis.handle(request) + apis.handle(request, requestLocal) } catch { case e: FatalExitError => - shutdownComplete.countDown() + completeShutdown() Exit.exit(e.statusCode) case e: Throwable => error("Exception when handling request", e) } finally { @@ -84,6 +85,11 @@ class KafkaRequestHandler(id: Int, case null => // continue } } + completeShutdown() + } + + private def completeShutdown(): Unit = { + requestLocal.close() shutdownComplete.countDown() } diff --git a/core/src/main/scala/kafka/server/PartitionMetadataFile.scala b/core/src/main/scala/kafka/server/PartitionMetadataFile.scala index 25b1ba6129d2b..c08c87dabe492 100644 --- a/core/src/main/scala/kafka/server/PartitionMetadataFile.scala +++ b/core/src/main/scala/kafka/server/PartitionMetadataFile.scala @@ -138,7 +138,9 @@ class PartitionMetadataFile(val file: File, file.exists() } - def delete(): Boolean = { - file.delete() + def delete(): Unit = { + Files.delete(file.toPath) } + + override def toString: String = s"PartitionMetadataFile(path=$path)" } diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index d813241bf3058..6ca8169bf3abe 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -21,7 +21,6 @@ import java.util.Optional import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.locks.Lock - import com.yammer.metrics.core.Meter import kafka.api._ import kafka.cluster.{BrokerEndPoint, Partition} @@ -606,11 +605,12 @@ class ReplicaManager(val config: KafkaConfig, entriesPerPartition: Map[TopicPartition, MemoryRecords], responseCallback: Map[TopicPartition, PartitionResponse] => Unit, delayedProduceLock: Option[Lock] = None, - recordConversionStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit = _ => ()): Unit = { + recordConversionStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit = _ => (), + requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { if (isValidRequiredAcks(requiredAcks)) { val sTime = time.milliseconds val localProduceResults = appendToLocalLog(internalTopicsAllowed = internalTopicsAllowed, - origin, entriesPerPartition, requiredAcks) + origin, entriesPerPartition, requiredAcks, requestLocal) debug("Produce to local log in %d ms".format(time.milliseconds - sTime)) val produceStatus = localProduceResults.map { case (topicPartition, result) => @@ -926,7 +926,8 @@ class ReplicaManager(val config: KafkaConfig, private def appendToLocalLog(internalTopicsAllowed: Boolean, origin: AppendOrigin, entriesPerPartition: Map[TopicPartition, MemoryRecords], - requiredAcks: Short): Map[TopicPartition, LogAppendResult] = { + requiredAcks: Short, + requestLocal: RequestLocal): Map[TopicPartition, LogAppendResult] = { val traceEnabled = isTraceEnabled def processFailedRecord(topicPartition: TopicPartition, t: Throwable) = { val logStartOffset = onlinePartition(topicPartition).map(_.logStartOffset).getOrElse(-1L) @@ -952,7 +953,7 @@ class ReplicaManager(val config: KafkaConfig, } else { try { val partition = getPartitionOrException(topicPartition) - val info = partition.appendRecordsToLeader(records, origin, requiredAcks) + val info = partition.appendRecordsToLeader(records, origin, requiredAcks, requestLocal) val numAppendedMessages = info.numMessages // update stats for successfully appended bytes and messages as bytesInRate and messageInRate diff --git a/core/src/main/scala/kafka/server/RequestLocal.scala b/core/src/main/scala/kafka/server/RequestLocal.scala new file mode 100644 index 0000000000000..5af495f8675f8 --- /dev/null +++ b/core/src/main/scala/kafka/server/RequestLocal.scala @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.server + +import org.apache.kafka.common.utils.BufferSupplier + +object RequestLocal { + val NoCaching: RequestLocal = RequestLocal(BufferSupplier.NO_CACHING) + + /** The returned instance should be confined to a single thread. */ + def withThreadConfinedCaching: RequestLocal = RequestLocal(BufferSupplier.create()) +} + +/** + * Container for stateful instances where the lifecycle is scoped to one request. + * + * When each request is handled by one thread, efficient data structures with no locking or atomic operations + * can be used (see RequestLocal.withThreadConfinedCaching). + */ +case class RequestLocal(bufferSupplier: BufferSupplier) { + def close(): Unit = bufferSupplier.close() +} diff --git a/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala b/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala index 6dfaa1800406c..70e44c89e10ea 100644 --- a/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala +++ b/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala @@ -21,7 +21,7 @@ import java.util.concurrent.TimeUnit import kafka.coordinator.group.GroupCoordinator import kafka.coordinator.transaction.TransactionCoordinator import kafka.metrics.KafkaMetricsGroup -import kafka.server.{RaftReplicaManager, RequestHandlerHelper} +import kafka.server.{RaftReplicaManager, RequestHandlerHelper, RequestLocal} import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.metadata.MetadataRecordType._ import org.apache.kafka.common.metadata._ @@ -248,7 +248,7 @@ class BrokerMetadataListener( case Some(topicName) => info(s"Processing deletion of topic $topicName with id ${record.topicId}") val removedPartitions = imageBuilder.partitionsBuilder().removeTopicById(record.topicId()) - groupCoordinator.handleDeletedPartitions(removedPartitions.map(_.toTopicPartition).toSeq) + groupCoordinator.handleDeletedPartitions(removedPartitions.map(_.toTopicPartition).toSeq, RequestLocal.NoCaching) configRepository.remove(new ConfigResource(ConfigResource.Type.TOPIC, topicName)) } } diff --git a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala index 91d5ecd3997ac..a9b471b1622ba 100644 --- a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala +++ b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala @@ -19,7 +19,7 @@ package kafka.tools import kafka.network.RequestChannel import kafka.raft.RaftManager -import kafka.server.{ApiRequestHandler, ApiVersionManager} +import kafka.server.{ApiRequestHandler, ApiVersionManager, RequestLocal} import kafka.utils.Logging import org.apache.kafka.common.internals.FatalExitError import org.apache.kafka.common.message.{BeginQuorumEpochResponseData, EndQuorumEpochResponseData, FetchResponseData, FetchSnapshotResponseData, VoteResponseData} @@ -37,7 +37,7 @@ class TestRaftRequestHandler( apiVersionManager: ApiVersionManager ) extends ApiRequestHandler with Logging { - override def handle(request: RequestChannel.Request): Unit = { + override def handle(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { try { trace(s"Handling request:${request.requestDesc(true)} with context ${request.context}") request.header.apiKey match { diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionLockTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionLockTest.scala index 8bb54c4ce070c..8dc37d4e4ad1b 100644 --- a/core/src/test/scala/unit/kafka/cluster/PartitionLockTest.scala +++ b/core/src/test/scala/unit/kafka/cluster/PartitionLockTest.scala @@ -20,7 +20,6 @@ package kafka.cluster import java.util.Properties import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent._ - import kafka.api.ApiVersion import kafka.log._ import kafka.server._ @@ -283,7 +282,7 @@ class PartitionLockTest extends Logging { val log = super.createLog(isNew, isFutureReplica, offsetCheckpoints, None) val logDirFailureChannel = new LogDirFailureChannel(1) val segments = new LogSegments(log.topicPartition) - val leaderEpochCache = Log.maybeCreateLeaderEpochCache(log.dir, log.topicPartition, logDirFailureChannel, log.config.messageFormatVersion.recordVersion) + val leaderEpochCache = Log.maybeCreateLeaderEpochCache(log.dir, log.topicPartition, logDirFailureChannel, log.config.messageFormatVersion.recordVersion, "") val maxProducerIdExpirationMs = 60 * 60 * 1000 val producerStateManager = new ProducerStateManager(log.topicPartition, log.dir, maxProducerIdExpirationMs) val offsets = LogLoader.load(LoadLogParams( @@ -336,10 +335,11 @@ class PartitionLockTest extends Logging { } private def append(partition: Partition, numRecords: Int, followerQueues: Seq[ArrayBlockingQueue[MemoryRecords]]): Unit = { + val requestLocal = RequestLocal.withThreadConfinedCaching (0 until numRecords).foreach { _ => val batch = TestUtils.records(records = List(new SimpleRecord("k1".getBytes, "v1".getBytes), new SimpleRecord("k2".getBytes, "v2".getBytes))) - partition.appendRecordsToLeader(batch, origin = AppendOrigin.Client, requiredAcks = 0) + partition.appendRecordsToLeader(batch, origin = AppendOrigin.Client, requiredAcks = 0, requestLocal) followerQueues.foreach(_.put(batch)) } } @@ -385,11 +385,12 @@ class PartitionLockTest extends Logging { leaderEpochCache, producerStateManager, logDirFailureChannel, - topicId = None, + _topicId = None, keepPartitionMetadataFile = true) { - override def appendAsLeader(records: MemoryRecords, leaderEpoch: Int, origin: AppendOrigin, interBrokerProtocolVersion: ApiVersion): LogAppendInfo = { - val appendInfo = super.appendAsLeader(records, leaderEpoch, origin, interBrokerProtocolVersion) + override def appendAsLeader(records: MemoryRecords, leaderEpoch: Int, origin: AppendOrigin, + interBrokerProtocolVersion: ApiVersion, requestLocal: RequestLocal): LogAppendInfo = { + val appendInfo = super.appendAsLeader(records, leaderEpoch, origin, interBrokerProtocolVersion, requestLocal) appendSemaphore.acquire() appendInfo } diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala index 8e492acb7438a..2b6d7ff8dd03f 100644 --- a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala +++ b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala @@ -72,7 +72,7 @@ class PartitionTest extends AbstractPartitionTest { assertEquals(17L, log.logEndOffset) val leaderEpoch = 10 - val partition = setupPartitionWithMocks(leaderEpoch = leaderEpoch, isLeader = true, log = log) + val partition = setupPartitionWithMocks(leaderEpoch = leaderEpoch, isLeader = true) def epochEndOffset(epoch: Int, endOffset: Long): FetchResponseData.EpochEndOffset = { new FetchResponseData.EpochEndOffset() @@ -143,7 +143,7 @@ class PartitionTest extends AbstractPartitionTest { ), leaderEpoch = 5) assertEquals(4, log.logEndOffset) - val partition = setupPartitionWithMocks(leaderEpoch = leaderEpoch, isLeader = true, log = log) + val partition = setupPartitionWithMocks(leaderEpoch = leaderEpoch, isLeader = true) assertEquals(Some(4), partition.leaderLogIfLocal.map(_.logEndOffset)) val epochEndOffset = partition.lastOffsetForLeaderEpoch(currentLeaderEpoch = Optional.of[Integer](leaderEpoch), @@ -171,7 +171,7 @@ class PartitionTest extends AbstractPartitionTest { ), leaderEpoch = 5) assertEquals(4, log.logEndOffset) - val partition = setupPartitionWithMocks(leaderEpoch = leaderEpoch, isLeader = true, log = log) + val partition = setupPartitionWithMocks(leaderEpoch = leaderEpoch, isLeader = true) assertEquals(Some(4), partition.leaderLogIfLocal.map(_.logEndOffset)) assertEquals(None, log.latestEpoch) @@ -236,7 +236,7 @@ class PartitionTest extends AbstractPartitionTest { val log = super.createLog(isNew, isFutureReplica, offsetCheckpoints, None) val logDirFailureChannel = new LogDirFailureChannel(1) val segments = new LogSegments(log.topicPartition) - val leaderEpochCache = Log.maybeCreateLeaderEpochCache(log.dir, log.topicPartition, logDirFailureChannel, log.config.messageFormatVersion.recordVersion) + val leaderEpochCache = Log.maybeCreateLeaderEpochCache(log.dir, log.topicPartition, logDirFailureChannel, log.config.messageFormatVersion.recordVersion, "") val maxProducerIdExpirationMs = 60 * 60 * 1000 val producerStateManager = new ProducerStateManager(log.topicPartition, log.dir, maxProducerIdExpirationMs) val offsets = LogLoader.load(LoadLogParams( @@ -592,10 +592,11 @@ class PartitionTest extends AbstractPartitionTest { assertEquals(leaderEpoch, partition.getLeaderEpoch, "Current leader epoch") assertEquals(Set[Integer](leader, follower2), partition.isrState.isr, "ISR") + val requestLocal = RequestLocal.withThreadConfinedCaching // after makeLeader(() call, partition should know about all the replicas // append records with initial leader epoch - partition.appendRecordsToLeader(batch1, origin = AppendOrigin.Client, requiredAcks = 0) - partition.appendRecordsToLeader(batch2, origin = AppendOrigin.Client, requiredAcks = 0) + partition.appendRecordsToLeader(batch1, origin = AppendOrigin.Client, requiredAcks = 0, requestLocal) + partition.appendRecordsToLeader(batch2, origin = AppendOrigin.Client, requiredAcks = 0, requestLocal) assertEquals(partition.localLogOrException.logStartOffset, partition.localLogOrException.highWatermark, "Expected leader's HW not move") @@ -733,8 +734,7 @@ class PartitionTest extends AbstractPartitionTest { } private def setupPartitionWithMocks(leaderEpoch: Int, - isLeader: Boolean, - log: Log = logManager.getOrCreateLog(topicPartition, topicId = None)): Partition = { + isLeader: Boolean): Partition = { partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None) val controllerEpoch = 0 @@ -840,7 +840,7 @@ class PartitionTest extends AbstractPartitionTest { new SimpleRecord("k2".getBytes, "v2".getBytes), new SimpleRecord("k3".getBytes, "v3".getBytes)), baseOffset = 0L) - partition.appendRecordsToLeader(records, origin = AppendOrigin.Client, requiredAcks = 0) + partition.appendRecordsToLeader(records, origin = AppendOrigin.Client, requiredAcks = 0, RequestLocal.withThreadConfinedCaching) def fetchLatestOffset(isolationLevel: Option[IsolationLevel]): TimestampAndOffset = { val res = partition.fetchOffsetForTimestamp(ListOffsetsRequest.LATEST_TIMESTAMP, @@ -955,11 +955,13 @@ class PartitionTest extends AbstractPartitionTest { assertEquals(leaderEpoch, partition.getLeaderEpoch, "Current leader epoch") assertEquals(Set[Integer](leader, follower2), partition.isrState.isr, "ISR") + val requestLocal = RequestLocal.withThreadConfinedCaching + // after makeLeader(() call, partition should know about all the replicas // append records with initial leader epoch val lastOffsetOfFirstBatch = partition.appendRecordsToLeader(batch1, origin = AppendOrigin.Client, - requiredAcks = 0).lastOffset - partition.appendRecordsToLeader(batch2, origin = AppendOrigin.Client, requiredAcks = 0) + requiredAcks = 0, requestLocal).lastOffset + partition.appendRecordsToLeader(batch2, origin = AppendOrigin.Client, requiredAcks = 0, requestLocal) assertEquals(partition.localLogOrException.logStartOffset, partition.log.get.highWatermark, "Expected leader's HW not move") // let the follower in ISR move leader's HW to move further but below LEO @@ -1000,7 +1002,7 @@ class PartitionTest extends AbstractPartitionTest { val currentLeaderEpochStartOffset = partition.localLogOrException.logEndOffset // append records with the latest leader epoch - partition.appendRecordsToLeader(batch3, origin = AppendOrigin.Client, requiredAcks = 0) + partition.appendRecordsToLeader(batch3, origin = AppendOrigin.Client, requiredAcks = 0, requestLocal) // fetch from follower not in ISR from log start offset should not add this follower to ISR updateFollowerFetchState(follower1, LogOffsetMetadata(0)) @@ -2046,7 +2048,7 @@ class PartitionTest extends AbstractPartitionTest { leaderEpochCache, producerStateManager, logDirFailureChannel, - topicId = None, + _topicId = None, keepPartitionMetadataFile = true) { override def appendAsFollower(records: MemoryRecords): LogAppendInfo = { diff --git a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala index 14e0f3fd7fd7e..6dfb396e9bad4 100644 --- a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala @@ -21,7 +21,6 @@ import java.util.concurrent.{ConcurrentHashMap, Executors} import java.util.{Collections, Random} import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.locks.Lock - import kafka.coordinator.AbstractCoordinatorConcurrencyTest._ import kafka.log.{AppendOrigin, Log} import kafka.server._ @@ -160,8 +159,10 @@ object AbstractCoordinatorConcurrencyTest { class TestReplicaManager extends ReplicaManager( null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, None, null, null) { + @volatile var logs: mutable.Map[TopicPartition, (Log, Long)] = _ var producePurgatory: DelayedOperationPurgatory[DelayedProduce] = _ var watchKeys: mutable.Set[TopicPartitionOperationKey] = _ + def createDelayedProducePurgatory(timer: MockTimer): Unit = { producePurgatory = new DelayedOperationPurgatory[DelayedProduce]("Produce", timer, 1, reaperEnabled = false) watchKeys = Collections.newSetFromMap(new ConcurrentHashMap[TopicPartitionOperationKey, java.lang.Boolean]()).asScala @@ -176,7 +177,8 @@ object AbstractCoordinatorConcurrencyTest { entriesPerPartition: Map[TopicPartition, MemoryRecords], responseCallback: Map[TopicPartition, PartitionResponse] => Unit, delayedProduceLock: Option[Lock] = None, - processingStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit = _ => ()): Unit = { + processingStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit = _ => (), + requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { if (entriesPerPartition.isEmpty) return @@ -204,20 +206,24 @@ object AbstractCoordinatorConcurrencyTest { watchKeys ++= producerRequestKeys producePurgatory.tryCompleteElseWatch(delayedProduce, producerRequestKeys) } + override def getMagic(topicPartition: TopicPartition): Option[Byte] = { Some(RecordBatch.MAGIC_VALUE_V2) } - @volatile var logs: mutable.Map[TopicPartition, (Log, Long)] = _ + def getOrCreateLogs(): mutable.Map[TopicPartition, (Log, Long)] = { if (logs == null) logs = mutable.Map[TopicPartition, (Log, Long)]() logs } + def updateLog(topicPartition: TopicPartition, log: Log, endOffset: Long): Unit = { getOrCreateLogs().put(topicPartition, (log, endOffset)) } + override def getLog(topicPartition: TopicPartition): Option[Log] = getOrCreateLogs().get(topicPartition).map(l => l._1) + override def getLogEndOffset(topicPartition: TopicPartition): Option[Long] = getOrCreateLogs().get(topicPartition).map(l => l._2) } diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala index 39826894c8f1a..23ebbebbb2cca 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala @@ -18,9 +18,8 @@ package kafka.coordinator.group import java.util.Optional - import kafka.common.OffsetAndMetadata -import kafka.server.{DelayedOperationPurgatory, HostedPartition, KafkaConfig, ReplicaManager} +import kafka.server.{DelayedOperationPurgatory, HostedPartition, KafkaConfig, ReplicaManager, RequestLocal} import kafka.utils._ import kafka.utils.timer.MockTimer import org.apache.kafka.common.TopicPartition @@ -29,9 +28,9 @@ import org.apache.kafka.common.record.{MemoryRecords, RecordBatch} import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse import org.apache.kafka.common.requests.{JoinGroupRequest, OffsetCommitRequest, OffsetFetchResponse, TransactionResult} import org.easymock.{Capture, EasyMock, IAnswer} + import java.util.concurrent.TimeUnit import java.util.concurrent.locks.ReentrantLock - import kafka.cluster.Partition import kafka.log.AppendOrigin import kafka.zk.KafkaZkClient @@ -3421,7 +3420,8 @@ class GroupCoordinatorTest { @Test def testDeleteOffsetOfNonExistingGroup(): Unit = { val tp = new TopicPartition("foo", 0) - val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp)) + val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp), + RequestLocal.NoCaching) assertEquals(Errors.GROUP_ID_NOT_FOUND, groupError) assertTrue(topics.isEmpty) @@ -3432,7 +3432,8 @@ class GroupCoordinatorTest { val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID dynamicJoinGroup(groupId, memberId, "My Protocol", protocols) val tp = new TopicPartition("foo", 0) - val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp)) + val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp), + RequestLocal.NoCaching) assertEquals(Errors.NON_EMPTY_GROUP, groupError) assertTrue(topics.isEmpty) @@ -3476,7 +3477,8 @@ class GroupCoordinatorTest { EasyMock.expect(replicaManager.onlinePartition(groupTopicPartition)).andStubReturn(Some(partition)) EasyMock.replay(replicaManager, partition) - val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0)) + val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0), + RequestLocal.NoCaching) assertEquals(Errors.NONE, groupError) assertEquals(1, topics.size) @@ -3505,7 +3507,8 @@ class GroupCoordinatorTest { Map(tp -> offset)) assertEquals(Errors.NONE, validOffsetCommitResult(tp)) - val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp)) + val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp), + RequestLocal.NoCaching) assertEquals(Errors.NONE, groupError) assertEquals(1, topics.size) @@ -3519,7 +3522,8 @@ class GroupCoordinatorTest { groupCoordinator.groupManager.addGroup(group) val tp = new TopicPartition("foo", 0) - val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp)) + val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(tp), + RequestLocal.NoCaching) assertEquals(Errors.GROUP_ID_NOT_FOUND, groupError) assertTrue(topics.isEmpty) @@ -3562,7 +3566,8 @@ class GroupCoordinatorTest { EasyMock.expect(replicaManager.onlinePartition(groupTopicPartition)).andStubReturn(Some(partition)) EasyMock.replay(replicaManager, partition) - val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0)) + val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0), + RequestLocal.NoCaching) assertEquals(Errors.NONE, groupError) assertEquals(1, topics.size) @@ -3609,7 +3614,8 @@ class GroupCoordinatorTest { EasyMock.expect(replicaManager.onlinePartition(groupTopicPartition)).andStubReturn(Some(partition)) EasyMock.replay(replicaManager, partition) - val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0, t2p0)) + val (groupError, topics) = groupCoordinator.handleDeleteOffsets(groupId, Seq(t1p0, t2p0), + RequestLocal.NoCaching) assertEquals(Errors.NONE, groupError) assertEquals(2, topics.size) @@ -3789,12 +3795,14 @@ class GroupCoordinatorTest { EasyMock.anyObject().asInstanceOf[Map[TopicPartition, MemoryRecords]], EasyMock.capture(capturedArgument), EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]], - EasyMock.anyObject())).andAnswer(new IAnswer[Unit] { - override def answer = capturedArgument.getValue.apply( + EasyMock.anyObject(), + EasyMock.anyObject() + )).andAnswer(new IAnswer[Unit] { + override def answer: Unit = capturedArgument.getValue.apply( Map(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId) -> new PartitionResponse(appendRecordError, 0L, RecordBatch.NO_TIMESTAMP, 0L) - ) - )}) + )) + }) EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andReturn(Some(RecordBatch.MAGIC_VALUE_V1)).anyTimes() EasyMock.replay(replicaManager) @@ -3821,6 +3829,7 @@ class GroupCoordinatorTest { EasyMock.anyObject().asInstanceOf[Map[TopicPartition, MemoryRecords]], EasyMock.capture(capturedArgument), EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]], + EasyMock.anyObject(), EasyMock.anyObject())).andAnswer(new IAnswer[Unit] { override def answer = capturedArgument.getValue.apply( Map(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId) -> @@ -3963,6 +3972,7 @@ class GroupCoordinatorTest { EasyMock.anyObject().asInstanceOf[Map[TopicPartition, MemoryRecords]], EasyMock.capture(capturedArgument), EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]], + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(new IAnswer[Unit] { override def answer = capturedArgument.getValue.apply( @@ -3996,6 +4006,7 @@ class GroupCoordinatorTest { EasyMock.anyObject().asInstanceOf[Map[TopicPartition, MemoryRecords]], EasyMock.capture(capturedArgument), EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]], + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(new IAnswer[Unit] { override def answer = capturedArgument.getValue.apply( diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala index 231382e515a6b..5b2152bc5785d 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala @@ -21,15 +21,15 @@ import java.lang.management.ManagementFactory import java.nio.ByteBuffer import java.util.concurrent.locks.ReentrantLock import java.util.{Collections, Optional} - import com.yammer.metrics.core.Gauge + import javax.management.ObjectName import kafka.api._ import kafka.cluster.Partition import kafka.common.OffsetAndMetadata import kafka.log.{AppendOrigin, Log, LogAppendInfo} import kafka.metrics.KafkaYammerMetrics -import kafka.server.{FetchDataInfo, FetchLogEnd, HostedPartition, KafkaConfig, LogOffsetMetadata, ReplicaManager} +import kafka.server.{FetchDataInfo, FetchLogEnd, HostedPartition, KafkaConfig, LogOffsetMetadata, ReplicaManager, RequestLocal} import kafka.utils.{KafkaScheduler, MockTime, TestUtils} import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription @@ -105,7 +105,7 @@ class GroupMetadataManagerTest { var expiredOffsets: Int = 0 var infoCount = 0 val gmm = new GroupMetadataManager(0, ApiVersion.latestVersion, offsetConfig, replicaManager, time, metrics) { - override def cleanupGroupMetadata(groups: Iterable[GroupMetadata], + override def cleanupGroupMetadata(groups: Iterable[GroupMetadata], requestLocal: RequestLocal, selector: GroupMetadata => Map[TopicPartition, OffsetAndMetadata]): Int = expiredOffsets override def info(msg: => String): Unit = infoCount += 1 @@ -1418,8 +1418,8 @@ class GroupMetadataManagerTest { EasyMock.reset(partition) EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]), - origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt())) - .andReturn(LogAppendInfo.UnknownLogAppendInfo) + origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt(), + EasyMock.anyObject())).andReturn(LogAppendInfo.UnknownLogAppendInfo) EasyMock.replay(partition) groupMetadataManager.cleanupGroupMetadata() @@ -1453,8 +1453,8 @@ class GroupMetadataManagerTest { EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andStubReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE)) mockGetPartition() EasyMock.expect(partition.appendRecordsToLeader(EasyMock.capture(recordsCapture), - origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt())) - .andReturn(LogAppendInfo.UnknownLogAppendInfo) + origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt(), + EasyMock.anyObject())).andReturn(LogAppendInfo.UnknownLogAppendInfo) EasyMock.replay(replicaManager, partition) groupMetadataManager.cleanupGroupMetadata() @@ -1501,8 +1501,8 @@ class GroupMetadataManagerTest { EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andStubReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE)) mockGetPartition() EasyMock.expect(partition.appendRecordsToLeader(EasyMock.capture(recordsCapture), - origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt())) - .andReturn(LogAppendInfo.UnknownLogAppendInfo) + origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt(), + EasyMock.anyObject())).andReturn(LogAppendInfo.UnknownLogAppendInfo) EasyMock.replay(replicaManager, partition) groupMetadataManager.cleanupGroupMetadata() @@ -1576,8 +1576,8 @@ class GroupMetadataManagerTest { val recordsCapture: Capture[MemoryRecords] = EasyMock.newCapture() EasyMock.expect(partition.appendRecordsToLeader(EasyMock.capture(recordsCapture), - origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt())) - .andReturn(LogAppendInfo.UnknownLogAppendInfo) + origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt(), + EasyMock.anyObject())).andReturn(LogAppendInfo.UnknownLogAppendInfo) EasyMock.replay(partition) groupMetadataManager.cleanupGroupMetadata() @@ -1677,8 +1677,8 @@ class GroupMetadataManagerTest { // expect the offset tombstone EasyMock.reset(partition) EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]), - origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt())) - .andReturn(LogAppendInfo.UnknownLogAppendInfo) + origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt(), + EasyMock.anyObject())).andReturn(LogAppendInfo.UnknownLogAppendInfo) EasyMock.replay(partition) groupMetadataManager.cleanupGroupMetadata() @@ -1701,8 +1701,8 @@ class GroupMetadataManagerTest { // expect the offset tombstone EasyMock.reset(partition) EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]), - origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt())) - .andReturn(LogAppendInfo.UnknownLogAppendInfo) + origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt(), + EasyMock.anyObject())).andReturn(LogAppendInfo.UnknownLogAppendInfo) EasyMock.replay(partition) groupMetadataManager.cleanupGroupMetadata() @@ -1744,8 +1744,8 @@ class GroupMetadataManagerTest { // expect the offset tombstone EasyMock.reset(partition) EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]), - origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt())) - .andReturn(LogAppendInfo.UnknownLogAppendInfo) + origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt(), + EasyMock.anyObject())).andReturn(LogAppendInfo.UnknownLogAppendInfo) EasyMock.replay(partition) groupMetadataManager.cleanupGroupMetadata() @@ -1822,8 +1822,8 @@ class GroupMetadataManagerTest { // expect the offset tombstone EasyMock.reset(partition) EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]), - origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt())) - .andReturn(LogAppendInfo.UnknownLogAppendInfo) + origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt(), + EasyMock.anyObject())).andReturn(LogAppendInfo.UnknownLogAppendInfo) EasyMock.replay(partition) groupMetadataManager.cleanupGroupMetadata() @@ -1948,8 +1948,8 @@ class GroupMetadataManagerTest { // expect the offset tombstone EasyMock.expect(partition.appendRecordsToLeader(EasyMock.anyObject(classOf[MemoryRecords]), - origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt())) - .andReturn(LogAppendInfo.UnknownLogAppendInfo) + origin = EasyMock.eq(AppendOrigin.Coordinator), requiredAcks = EasyMock.anyInt(), + EasyMock.anyObject())).andReturn(LogAppendInfo.UnknownLogAppendInfo) EasyMock.expectLastCall().times(1) EasyMock.replay(partition) @@ -2270,6 +2270,7 @@ class GroupMetadataManagerTest { EasyMock.anyObject().asInstanceOf[Map[TopicPartition, MemoryRecords]], EasyMock.capture(capturedArgument), EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]], + EasyMock.anyObject(), EasyMock.anyObject()) ) EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andStubReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE)) @@ -2286,6 +2287,7 @@ class GroupMetadataManagerTest { EasyMock.capture(capturedRecords), EasyMock.capture(capturedCallback), EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]], + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(new IAnswer[Unit] { override def answer = capturedCallback.getValue.apply( diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala index e1786d0ee21ff..e02c2fe3c320b 100644 --- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorConcurrencyTest.scala @@ -18,12 +18,11 @@ package kafka.coordinator.transaction import java.nio.ByteBuffer import java.util.concurrent.atomic.AtomicBoolean - import kafka.coordinator.AbstractCoordinatorConcurrencyTest import kafka.coordinator.AbstractCoordinatorConcurrencyTest._ import kafka.coordinator.transaction.TransactionCoordinatorConcurrencyTest._ import kafka.log.Log -import kafka.server.{FetchDataInfo, FetchLogEnd, KafkaConfig, LogOffsetMetadata, MetadataCache} +import kafka.server.{FetchDataInfo, FetchLogEnd, KafkaConfig, LogOffsetMetadata, MetadataCache, RequestLocal} import kafka.utils.{Pool, TestUtils} import org.apache.kafka.clients.{ClientResponse, NetworkClient} import org.apache.kafka.common.internals.Topic.TRANSACTION_STATE_TOPIC_NAME @@ -509,7 +508,8 @@ class TransactionCoordinatorConcurrencyTest extends AbstractCoordinatorConcurren class InitProducerIdOperation(val producerIdAndEpoch: Option[ProducerIdAndEpoch] = None) extends TxnOperation[InitProducerIdResult] { override def run(txn: Transaction): Unit = { - transactionCoordinator.handleInitProducerId(txn.transactionalId, 60000, producerIdAndEpoch, resultCallback) + transactionCoordinator.handleInitProducerId(txn.transactionalId, 60000, producerIdAndEpoch, resultCallback, + RequestLocal.withThreadConfinedCaching) replicaManager.tryCompleteActions() } override def awaitAndVerify(txn: Transaction): Unit = { @@ -526,7 +526,8 @@ class TransactionCoordinatorConcurrencyTest extends AbstractCoordinatorConcurren txnMetadata.producerId, txnMetadata.producerEpoch, partitions, - resultCallback) + resultCallback, + RequestLocal.withThreadConfinedCaching) replicaManager.tryCompleteActions() } } @@ -544,7 +545,8 @@ class TransactionCoordinatorConcurrencyTest extends AbstractCoordinatorConcurren txnMetadata.producerId, txnMetadata.producerEpoch, transactionResult(txn), - resultCallback) + resultCallback, + RequestLocal.withThreadConfinedCaching) } } override def awaitAndVerify(txn: Transaction): Unit = { diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorTest.scala index f6b5e54dfe19a..38e8e711975ef 100644 --- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorTest.scala @@ -118,6 +118,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.anyObject().asInstanceOf[TxnTransitMetadata], EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => capturedErrorsCallback.getValue.apply(Errors.NONE)).anyTimes() EasyMock.replay(pidGenerator, transactionManager) @@ -145,6 +146,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.anyObject().asInstanceOf[TxnTransitMetadata], EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => capturedErrorsCallback.getValue.apply(Errors.NONE)).anyTimes() EasyMock.replay(pidGenerator, transactionManager) @@ -169,6 +171,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.anyObject().asInstanceOf[TxnTransitMetadata], EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject() )).andAnswer(() => capturedErrorsCallback.getValue.apply(Errors.NONE)) @@ -314,6 +317,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.anyObject().asInstanceOf[TxnTransitMetadata], EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject() )) @@ -572,6 +576,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.eq(originalMetadata.prepareAbortOrCommit(PrepareAbort, time.milliseconds())), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => capturedErrorsCallback.getValue.apply(Errors.NONE)) @@ -641,6 +646,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.eq(txnTransitMetadata), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => { capturedErrorsCallback.getValue.apply(Errors.NOT_ENOUGH_REPLICAS) @@ -652,6 +658,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.eq(txnTransitMetadata), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => { capturedErrorsCallback.getValue.apply(Errors.NONE) @@ -724,6 +731,7 @@ class TransactionCoordinatorTest { txnStartTimestamp = time.milliseconds(), txnLastUpdateTimestamp = time.milliseconds())), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => capturedErrorsCallback.getValue.apply(Errors.NONE)) @@ -790,6 +798,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.anyObject().asInstanceOf[TxnTransitMetadata], EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => { capturedErrorsCallback.getValue.apply(Errors.NONE) @@ -827,6 +836,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.capture(capturedTxnTransitMetadata), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => { capturedErrorsCallback.getValue.apply(Errors.NONE) @@ -867,6 +877,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.capture(capturedTxnTransitMetadata), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => { capturedErrorsCallback.getValue.apply(Errors.NONE) @@ -910,6 +921,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.capture(capturedTxnTransitMetadata), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => { capturedErrorsCallback.getValue.apply(Errors.NONE) @@ -963,6 +975,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.eq(expectedTransition), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => {}).once() @@ -1046,6 +1059,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.eq(expectedTransition), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => capturedErrorsCallback.getValue.apply(Errors.NOT_ENOUGH_REPLICAS)).once() @@ -1179,6 +1193,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.capture(capturedNewMetadata), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject() )).andAnswer(() => { metadata.completeTransitionTo(capturedNewMetadata.getValue) @@ -1213,6 +1228,7 @@ class TransactionCoordinatorTest { EasyMock.eq(coordinatorEpoch), EasyMock.eq(transition), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => { if (runCallback) diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMarkerChannelManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMarkerChannelManagerTest.scala index 8aa07c6c627ce..0a0ec511535d0 100644 --- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMarkerChannelManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMarkerChannelManagerTest.scala @@ -107,6 +107,7 @@ class TransactionMarkerChannelManagerTest { EasyMock.eq(coordinatorEpoch), EasyMock.eq(expectedTransition), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject())) .andAnswer(() => { txnMetadata2.completeTransitionTo(expectedTransition) @@ -345,6 +346,7 @@ class TransactionMarkerChannelManagerTest { EasyMock.eq(coordinatorEpoch), EasyMock.eq(txnTransitionMetadata2), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject())) .andAnswer(() => { txnMetadata2.completeTransitionTo(txnTransitionMetadata2) @@ -392,6 +394,7 @@ class TransactionMarkerChannelManagerTest { EasyMock.eq(coordinatorEpoch), EasyMock.eq(txnTransitionMetadata2), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject())) .andAnswer(() => { txnMetadata2.pendingState = None @@ -439,6 +442,7 @@ class TransactionMarkerChannelManagerTest { EasyMock.eq(coordinatorEpoch), EasyMock.eq(txnTransitionMetadata2), EasyMock.capture(capturedErrorsCallback), + EasyMock.anyObject(), EasyMock.anyObject())) .andAnswer(() => capturedErrorsCallback.getValue.apply(Errors.COORDINATOR_NOT_AVAILABLE)) .andAnswer(() => { diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala index df576931525d6..410d6e2a26d06 100644 --- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala @@ -22,7 +22,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.locks.ReentrantLock import javax.management.ObjectName import kafka.log.{AppendOrigin, Log} -import kafka.server.{FetchDataInfo, FetchLogEnd, LogOffsetMetadata, ReplicaManager} +import kafka.server.{FetchDataInfo, FetchLogEnd, LogOffsetMetadata, ReplicaManager, RequestLocal} import kafka.utils.{MockScheduler, Pool, TestUtils} import kafka.zk.KafkaZkClient import org.apache.kafka.common.TopicPartition @@ -319,7 +319,7 @@ class TransactionStateManagerTest { new TopicPartition("topic1", 1)), time.milliseconds()) // append the new metadata into log - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch, newMetadata, assertCallback) + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch, newMetadata, assertCallback, requestLocal = RequestLocal.withThreadConfinedCaching) assertEquals(Right(Some(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata1))), transactionManager.getTransactionState(transactionalId1)) assertTrue(txnMetadata1.pendingState.isEmpty) @@ -334,25 +334,26 @@ class TransactionStateManagerTest { var failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds()) prepareForTxnMessageAppend(Errors.UNKNOWN_TOPIC_OR_PARTITION) - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback) + val requestLocal = RequestLocal.withThreadConfinedCaching + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback, requestLocal = requestLocal) assertEquals(Right(Some(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata1))), transactionManager.getTransactionState(transactionalId1)) assertTrue(txnMetadata1.pendingState.isEmpty) failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds()) prepareForTxnMessageAppend(Errors.NOT_ENOUGH_REPLICAS) - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback) + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback, requestLocal = requestLocal) assertEquals(Right(Some(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata1))), transactionManager.getTransactionState(transactionalId1)) assertTrue(txnMetadata1.pendingState.isEmpty) failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds()) prepareForTxnMessageAppend(Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND) - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback) + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback, requestLocal = requestLocal) assertEquals(Right(Some(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata1))), transactionManager.getTransactionState(transactionalId1)) assertTrue(txnMetadata1.pendingState.isEmpty) failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds()) prepareForTxnMessageAppend(Errors.REQUEST_TIMED_OUT) - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback) + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback, requestLocal = requestLocal) assertEquals(Right(Some(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata1))), transactionManager.getTransactionState(transactionalId1)) assertTrue(txnMetadata1.pendingState.isEmpty) } @@ -366,25 +367,26 @@ class TransactionStateManagerTest { var failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds()) prepareForTxnMessageAppend(Errors.NOT_LEADER_OR_FOLLOWER) - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback) + val requestLocal = RequestLocal.withThreadConfinedCaching + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback, requestLocal = requestLocal) assertEquals(Right(Some(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata1))), transactionManager.getTransactionState(transactionalId1)) assertTrue(txnMetadata1.pendingState.isEmpty) failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds()) prepareForTxnMessageAppend(Errors.NONE) transactionManager.removeTransactionsForTxnTopicPartition(partitionId, coordinatorEpoch) - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback) + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback, requestLocal = requestLocal) prepareForTxnMessageAppend(Errors.NONE) transactionManager.removeTransactionsForTxnTopicPartition(partitionId, coordinatorEpoch) transactionManager.addLoadedTransactionsToCache(partitionId, coordinatorEpoch + 1, new Pool[String, TransactionMetadata]()) transactionManager.putTransactionStateIfNotExists(txnMetadata1) - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback) + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback, requestLocal = requestLocal) prepareForTxnMessageAppend(Errors.NONE) transactionManager.removeTransactionsForTxnTopicPartition(partitionId, coordinatorEpoch) transactionManager.addLoadedTransactionsToCache(partitionId, coordinatorEpoch, new Pool[String, TransactionMetadata]()) - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback) + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback, requestLocal = requestLocal) } @Test @@ -398,7 +400,7 @@ class TransactionStateManagerTest { prepareForTxnMessageAppend(Errors.NONE) transactionManager.removeTransactionsForTxnTopicPartition(partitionId, coordinatorEpoch) transactionManager.addLoadingPartition(partitionId, coordinatorEpoch + 1) - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback) + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback, requestLocal = RequestLocal.withThreadConfinedCaching) } @Test @@ -410,13 +412,14 @@ class TransactionStateManagerTest { var failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds()) prepareForTxnMessageAppend(Errors.MESSAGE_TOO_LARGE) - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback) + val requestLocal = RequestLocal.withThreadConfinedCaching + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback, requestLocal = requestLocal) assertEquals(Right(Some(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata1))), transactionManager.getTransactionState(transactionalId1)) assertTrue(txnMetadata1.pendingState.isEmpty) failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds()) prepareForTxnMessageAppend(Errors.RECORD_LIST_TOO_LARGE) - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback) + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback, requestLocal = requestLocal) assertEquals(Right(Some(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata1))), transactionManager.getTransactionState(transactionalId1)) assertTrue(txnMetadata1.pendingState.isEmpty) } @@ -430,7 +433,7 @@ class TransactionStateManagerTest { val failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds()) prepareForTxnMessageAppend(Errors.UNKNOWN_TOPIC_OR_PARTITION) - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback, _ => true) + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback, _ => true, RequestLocal.withThreadConfinedCaching) assertEquals(Right(Some(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata1))), transactionManager.getTransactionState(transactionalId1)) assertEquals(Some(Ongoing), txnMetadata1.pendingState) } @@ -452,7 +455,7 @@ class TransactionStateManagerTest { txnMetadata1.producerEpoch = (txnMetadata1.producerEpoch + 1).toShort // append the new metadata into log - transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, newMetadata, assertCallback) + transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, newMetadata, assertCallback, requestLocal = RequestLocal.withThreadConfinedCaching) } @Test @@ -472,7 +475,7 @@ class TransactionStateManagerTest { // append the new metadata into log assertThrows(classOf[IllegalStateException], () => transactionManager.appendTransactionToLog(transactionalId1, - coordinatorEpoch = 10, newMetadata, assertCallback)) + coordinatorEpoch = 10, newMetadata, assertCallback, requestLocal = RequestLocal.withThreadConfinedCaching)) } @Test @@ -719,6 +722,7 @@ class TransactionStateManagerTest { EasyMock.eq(recordsByPartition), EasyMock.capture(capturedArgument), EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]], + EasyMock.anyObject(), EasyMock.anyObject() )).andAnswer(() => capturedArgument.getValue.apply( Map(partition -> new PartitionResponse(error, 0L, RecordBatch.NO_TIMESTAMP, 0L))) @@ -824,6 +828,7 @@ class TransactionStateManagerTest { EasyMock.anyObject().asInstanceOf[Map[TopicPartition, MemoryRecords]], EasyMock.capture(capturedArgument), EasyMock.anyObject().asInstanceOf[Option[ReentrantLock]], + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => capturedArgument.getValue.apply( Map(new TopicPartition(TRANSACTION_STATE_TOPIC_NAME, partitionId) -> diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala index 3cb32c0724e09..19de8ea2703bb 100644 --- a/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogCleanerManagerTest.scala @@ -100,8 +100,8 @@ class LogCleanerManagerTest extends Logging { val config = createLowRetentionLogConfig(logSegmentSize, LogConfig.Compact) val maxProducerIdExpirationMs = 60 * 60 * 1000 val segments = new LogSegments(tp) - val leaderEpochCache = Log.maybeCreateLeaderEpochCache(tpDir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion) - val producerStateManager = new ProducerStateManager(topicPartition, tpDir, maxProducerIdExpirationMs) + val leaderEpochCache = Log.maybeCreateLeaderEpochCache(tpDir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion, "") + val producerStateManager = new ProducerStateManager(topicPartition, tpDir, maxProducerIdExpirationMs, time) val offsets = LogLoader.load(LoadLogParams( tpDir, tp, @@ -121,7 +121,7 @@ class LogCleanerManagerTest extends Logging { extends Log(dir, config, segments, offsets.logStartOffset, offsets.recoveryPoint, offsets.nextOffsetMetadata, time.scheduler, new BrokerTopicStats, time, LogManager.ProducerIdExpirationCheckIntervalMs, topicPartition, leaderEpochCache, - producerStateManager, logDirFailureChannel, topicId = None, keepPartitionMetadataFile = true) { + producerStateManager, logDirFailureChannel, _topicId = None, keepPartitionMetadataFile = true) { // Throw an error in getFirstBatchTimestampForSegments since it is called in grabFilthiestLog() override def getFirstBatchTimestampForSegments(segments: Iterable[LogSegment]): Iterable[Long] = throw new IllegalStateException("Error!") diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala index 99ff1aac0a77b..5c91041b9f942 100755 --- a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala @@ -105,8 +105,8 @@ class LogCleanerTest { val logDirFailureChannel = new LogDirFailureChannel(10) val maxProducerIdExpirationMs = 60 * 60 * 1000 val logSegments = new LogSegments(topicPartition) - val leaderEpochCache = Log.maybeCreateLeaderEpochCache(dir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion) - val producerStateManager = new ProducerStateManager(topicPartition, dir, maxProducerIdExpirationMs) + val leaderEpochCache = Log.maybeCreateLeaderEpochCache(dir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion, "") + val producerStateManager = new ProducerStateManager(topicPartition, dir, maxProducerIdExpirationMs, time) val offsets = LogLoader.load(LoadLogParams( dir, topicPartition, @@ -136,7 +136,7 @@ class LogCleanerTest { leaderEpochCache = leaderEpochCache, producerStateManager = producerStateManager, logDirFailureChannel = logDirFailureChannel, - topicId = None, + _topicId = None, keepPartitionMetadataFile = true) { override def replaceSegments(newSegments: Seq[LogSegment], oldSegments: Seq[LogSegment], isRecoveredSwapFile: Boolean = false): Unit = { deleteStartLatch.countDown() diff --git a/core/src/test/scala/unit/kafka/log/LogLoaderTest.scala b/core/src/test/scala/unit/kafka/log/LogLoaderTest.scala index 0546db4db7ea6..9a7b627492472 100644 --- a/core/src/test/scala/unit/kafka/log/LogLoaderTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogLoaderTest.scala @@ -94,8 +94,8 @@ class LogLoaderTest { val logDirFailureChannel: LogDirFailureChannel = new LogDirFailureChannel(1) val maxProducerIdExpirationMs = 60 * 60 * 1000 val segments = new LogSegments(topicPartition) - val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion) - val producerStateManager = new ProducerStateManager(topicPartition, logDir, maxProducerIdExpirationMs) + val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion, "") + val producerStateManager = new ProducerStateManager(topicPartition, logDir, maxProducerIdExpirationMs, time) val loadLogParams = LoadLogParams(logDir, topicPartition, config, time.scheduler, time, logDirFailureChannel, hadCleanShutdown, segments, logStartOffset, logRecoveryPoint, maxProducerIdExpirationMs, leaderEpochCache, producerStateManager) @@ -264,8 +264,8 @@ class LogLoaderTest { super.add(wrapper) } } - val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, logConfig.messageFormatVersion.recordVersion) - val producerStateManager = new ProducerStateManager(topicPartition, logDir, maxProducerIdExpirationMs) + val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, logConfig.messageFormatVersion.recordVersion, "") + val producerStateManager = new ProducerStateManager(topicPartition, logDir, maxProducerIdExpirationMs, mockTime) val loadLogParams = LoadLogParams( logDir, topicPartition, @@ -284,7 +284,7 @@ class LogLoaderTest { new Log(logDir, logConfig, interceptedLogSegments, offsets.logStartOffset, offsets.recoveryPoint, offsets.nextOffsetMetadata, mockTime.scheduler, brokerTopicStats, mockTime, LogManager.ProducerIdExpirationCheckIntervalMs, topicPartition, leaderEpochCache, - producerStateManager, logDirFailureChannel, topicId = None, keepPartitionMetadataFile = true) + producerStateManager, logDirFailureChannel, _topicId = None, keepPartitionMetadataFile = true) } // Retain snapshots for the last 2 segments @@ -337,7 +337,7 @@ class LogLoaderTest { val config = LogConfig(new Properties()) val maxProducerIdExpirationMs = 300000 val segments = new LogSegments(topicPartition) - val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion) + val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion, "") val offsets = LogLoader.load(LoadLogParams( logDir, topicPartition, @@ -366,7 +366,7 @@ class LogLoaderTest { leaderEpochCache = leaderEpochCache, producerStateManager = stateManager, logDirFailureChannel = logDirFailureChannel, - topicId = None, + _topicId = None, keepPartitionMetadataFile = true) EasyMock.verify(stateManager) @@ -471,7 +471,7 @@ class LogLoaderTest { val maxProducerIdExpirationMs = 300000 val logDirFailureChannel = null val segments = new LogSegments(topicPartition) - val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion) + val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion, "") val offsets = LogLoader.load(LoadLogParams( logDir, topicPartition, @@ -500,7 +500,7 @@ class LogLoaderTest { leaderEpochCache = leaderEpochCache, producerStateManager = stateManager, logDirFailureChannel = logDirFailureChannel, - topicId = None, + _topicId = None, keepPartitionMetadataFile = true) EasyMock.verify(stateManager) @@ -532,7 +532,7 @@ class LogLoaderTest { val maxProducerIdExpirationMs = 300000 val logDirFailureChannel = null val segments = new LogSegments(topicPartition) - val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion) + val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion, "") val offsets = LogLoader.load(LoadLogParams( logDir, topicPartition, @@ -561,7 +561,7 @@ class LogLoaderTest { leaderEpochCache = leaderEpochCache, producerStateManager = stateManager, logDirFailureChannel = logDirFailureChannel, - topicId = None, + _topicId = None, keepPartitionMetadataFile = true) EasyMock.verify(stateManager) @@ -595,7 +595,7 @@ class LogLoaderTest { val maxProducerIdExpirationMs = 300000 val logDirFailureChannel = null val segments = new LogSegments(topicPartition) - val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion) + val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, config.messageFormatVersion.recordVersion, "") val offsets = LogLoader.load(LoadLogParams( logDir, topicPartition, @@ -624,7 +624,7 @@ class LogLoaderTest { leaderEpochCache = leaderEpochCache, producerStateManager = stateManager, logDirFailureChannel = logDirFailureChannel, - topicId = None, + _topicId = None, keepPartitionMetadataFile = true) EasyMock.verify(stateManager) diff --git a/core/src/test/scala/unit/kafka/log/LogSegmentsTest.scala b/core/src/test/scala/unit/kafka/log/LogSegmentsTest.scala index b929b9c4b8a52..9d0765aed687e 100644 --- a/core/src/test/scala/unit/kafka/log/LogSegmentsTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogSegmentsTest.scala @@ -46,7 +46,7 @@ class LogSegmentsTest { Utils.delete(logDir) } - private def assertEntry(segment: LogSegment, tested: java.util.Map.Entry[java.lang.Long, LogSegment]): Unit = { + private def assertEntry(segment: LogSegment, tested: java.util.Map.Entry[Long, LogSegment]): Unit = { assertEquals(segment.baseOffset, tested.getKey()) assertEquals(segment, tested.getValue()) } @@ -158,17 +158,13 @@ class LogSegmentsTest { List(seg1, seg2, seg3, seg4).foreach(segments.add) - // Test floorSegment, floorEntry + // Test floorSegment assertEquals(Some(seg1), segments.floorSegment(2)) - assertEntry(seg1, segments.floorEntry(2).get) assertEquals(Some(seg2), segments.floorSegment(3)) - assertEntry(seg2, segments.floorEntry(3).get) - // Test lowerSegment, lowerEntry + // Test lowerSegment assertEquals(Some(seg1), segments.lowerSegment(3)) - assertEntry(seg1, segments.lowerEntry(3).get) assertEquals(Some(seg2), segments.lowerSegment(4)) - assertEntry(seg2, segments.lowerEntry(4).get) // Test higherSegment, higherEntry assertEquals(Some(seg3), segments.higherSegment(4)) @@ -178,4 +174,53 @@ class LogSegmentsTest { segments.close() } + + @Test + def testHigherSegments(): Unit = { + val segments = new LogSegments(topicPartition) + + val seg1 = createSegment(1) + val seg2 = createSegment(3) + val seg3 = createSegment(5) + val seg4 = createSegment(7) + val seg5 = createSegment(9) + + List(seg1, seg2, seg3, seg4, seg5).foreach(segments.add) + + // higherSegments(0) should return all segments in order + { + val iterator = segments.higherSegments(0).iterator + List(seg1, seg2, seg3, seg4, seg5).foreach { + segment => + assertTrue(iterator.hasNext) + assertEquals(segment, iterator.next()) + } + assertFalse(iterator.hasNext) + } + + // higherSegments(1) should return all segments in order except seg1 + { + val iterator = segments.higherSegments(1).iterator + List(seg2, seg3, seg4, seg5).foreach { + segment => + assertTrue(iterator.hasNext) + assertEquals(segment, iterator.next()) + } + assertFalse(iterator.hasNext) + } + + // higherSegments(8) should return only seg5 + { + val iterator = segments.higherSegments(8).iterator + assertTrue(iterator.hasNext) + assertEquals(seg5, iterator.next()) + assertFalse(iterator.hasNext) + } + + // higherSegments(9) should return no segments + { + val iterator = segments.higherSegments(9).iterator + assertFalse(iterator.hasNext) + } + } } diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala index 30541165a8289..4cc88d0f5873c 100755 --- a/core/src/test/scala/unit/kafka/log/LogTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogTest.scala @@ -2319,9 +2319,8 @@ class LogTest { val log = createLog(logDir, logConfig) // Write a topic ID to the partition metadata file to ensure it is transferred correctly. - val id = Uuid.randomUuid() - log.topicId = Some(id) - log.partitionMetadataFile.write(id) + val topicId = Uuid.randomUuid() + log.assignTopicId(topicId) log.appendAsLeader(TestUtils.records(List(new SimpleRecord("foo".getBytes()))), leaderEpoch = 5) assertEquals(Some(5), log.latestEpoch) @@ -2336,8 +2335,8 @@ class LogTest { // Check the topic ID remains in memory and was copied correctly. assertTrue(log.topicId.isDefined) - assertEquals(id, log.topicId.get) - assertEquals(id, log.partitionMetadataFile.read().topicId) + assertEquals(topicId, log.topicId.get) + assertEquals(topicId, log.partitionMetadataFile.read().topicId) } @Test diff --git a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala index b0d4e3dc7efa3..af585bfd49605 100644 --- a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala @@ -18,13 +18,12 @@ package kafka.log import java.nio.ByteBuffer import java.util.concurrent.TimeUnit - import kafka.api.{ApiVersion, KAFKA_2_0_IV1, KAFKA_2_3_IV1} import kafka.common.{LongRef, RecordValidationException} import kafka.log.LogValidator.ValidationAndOffsetAssignResult import kafka.message._ import kafka.metrics.KafkaYammerMetrics -import kafka.server.BrokerTopicStats +import kafka.server.{BrokerTopicStats, RequestLocal} import kafka.utils.TestUtils.meterCount import org.apache.kafka.common.errors.{InvalidTimestampException, UnsupportedCompressionTypeException, UnsupportedForMessageFormatException} import org.apache.kafka.common.record._ @@ -128,8 +127,8 @@ class LogValidatorTest { RecordBatch.NO_PRODUCER_EPOCH, origin = AppendOrigin.Client, KAFKA_2_3_IV1, - brokerTopicStats - ) + brokerTopicStats, + RequestLocal.withThreadConfinedCaching) } @Test @@ -160,7 +159,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) val validatedRecords = validatedResults.validatedRecords assertEquals(records.records.asScala.size, validatedRecords.records.asScala.size, "message set size should not change") validatedRecords.batches.forEach(batch => validateLogAppendTime(now, 1234L, batch)) @@ -199,7 +199,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) val validatedRecords = validatedResults.validatedRecords assertEquals(records.records.asScala.size, validatedRecords.records.asScala.size, @@ -247,7 +248,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) val validatedRecords = validatedResults.validatedRecords assertEquals(records.records.asScala.size, validatedRecords.records.asScala.size, @@ -309,7 +311,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) } @Test @@ -353,7 +356,8 @@ class LogValidatorTest { partitionLeaderEpoch = partitionLeaderEpoch, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) val validatedRecords = validatingResults.validatedRecords var i = 0 @@ -425,7 +429,8 @@ class LogValidatorTest { partitionLeaderEpoch = partitionLeaderEpoch, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) val validatedRecords = validatingResults.validatedRecords var i = 0 @@ -481,7 +486,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) val validatedRecords = validatedResults.validatedRecords for (batch <- validatedRecords.batches.asScala) { @@ -526,7 +532,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) val validatedRecords = validatedResults.validatedRecords for (batch <- validatedRecords.batches.asScala) { @@ -583,7 +590,8 @@ class LogValidatorTest { partitionLeaderEpoch = partitionLeaderEpoch, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) val validatedRecords = validatedResults.validatedRecords var i = 0 @@ -636,7 +644,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats)) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching)) } @Test @@ -659,7 +668,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats)) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching)) } @Test @@ -682,7 +692,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats)) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching)) } @Test @@ -705,7 +716,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats)) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching)) } @Test @@ -727,7 +739,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords, offset) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords, offset) } @Test @@ -749,7 +762,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords, offset) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords, offset) } @Test @@ -772,7 +786,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords checkOffsets(messageWithOffset, offset) } @@ -796,7 +811,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords checkOffsets(messageWithOffset, offset) } @@ -821,7 +837,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords checkOffsets(compressedMessagesWithOffset, offset) } @@ -846,7 +863,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords checkOffsets(compressedMessagesWithOffset, offset) } @@ -869,7 +887,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) checkOffsets(validatedResults.validatedRecords, offset) verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records, compressed = false) @@ -894,7 +913,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) checkOffsets(validatedResults.validatedRecords, offset) verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records, compressed = false) @@ -919,7 +939,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) checkOffsets(validatedResults.validatedRecords, offset) verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records, compressed = true) @@ -944,7 +965,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) checkOffsets(validatedResults.validatedRecords, offset) verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records, compressed = true) @@ -969,7 +991,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats)) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching)) } @Test @@ -991,7 +1014,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Coordinator, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) val batches = TestUtils.toList(result.validatedRecords.batches) assertEquals(1, batches.size) val batch = batches.get(0) @@ -1018,7 +1042,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords, offset) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords, offset) } @Test @@ -1041,7 +1066,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords, offset) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords, offset) } @Test @@ -1063,7 +1089,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords, offset) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords, offset) } @Test @@ -1085,7 +1112,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords, offset) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords, offset) } @Test @@ -1108,7 +1136,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords, offset) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords, offset) } @Test @@ -1131,7 +1160,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords, offset) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords, offset) } @Test @@ -1156,7 +1186,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats)) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching)) } @Test @@ -1181,7 +1212,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats)) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching)) } @Test @@ -1204,7 +1236,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords, offset) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords, offset) } @Test @@ -1227,7 +1260,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats).validatedRecords, offset) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching).validatedRecords, offset) } @Test @@ -1248,7 +1282,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) ) assertEquals(metricsKeySet.count(_.getMBeanName.endsWith(s"${BrokerTopicStats.InvalidOffsetOrSequenceRecordsPerSec}")), 1) assertTrue(meterCount(s"${BrokerTopicStats.InvalidOffsetOrSequenceRecordsPerSec}") > 0) @@ -1278,7 +1313,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = KAFKA_2_0_IV1, - brokerTopicStats = brokerTopicStats)) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching)) } @Test @@ -1312,7 +1348,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching) ) assertTrue(e.invalidException.isInstanceOf[InvalidTimestampException]) @@ -1390,7 +1427,8 @@ class LogValidatorTest { partitionLeaderEpoch = RecordBatch.NO_PARTITION_LEADER_EPOCH, origin = AppendOrigin.Client, interBrokerProtocolVersion = ApiVersion.latestVersion, - brokerTopicStats = brokerTopicStats)) + brokerTopicStats = brokerTopicStats, + requestLocal = RequestLocal.withThreadConfinedCaching)) } private def createRecords(magicValue: Byte, diff --git a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala index 78510e388e8e5..5f81f59eddacf 100644 --- a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala +++ b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala @@ -46,7 +46,7 @@ class ProducerStateManagerTest { @BeforeEach def setUp(): Unit = { logDir = TestUtils.tempDir() - stateManager = new ProducerStateManager(partition, logDir, maxPidExpirationMs) + stateManager = new ProducerStateManager(partition, logDir, maxPidExpirationMs, time) } @AfterEach @@ -467,7 +467,7 @@ class ProducerStateManagerTest { append(stateManager, producerId, epoch, 1, 1L, isTransactional = true) stateManager.takeSnapshot() - val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs) + val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs, time) recoveredMapping.truncateAndReload(0L, 3L, time.milliseconds) // The snapshot only persists the last appended batch metadata @@ -490,7 +490,7 @@ class ProducerStateManagerTest { appendEndTxnMarker(stateManager, producerId, epoch, ControlRecordType.ABORT, offset = 2L) stateManager.takeSnapshot() - val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs) + val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs, time) recoveredMapping.truncateAndReload(0L, 3L, time.milliseconds) // The snapshot only persists the last appended batch metadata @@ -510,7 +510,7 @@ class ProducerStateManagerTest { offset = 0L, timestamp = appendTimestamp) stateManager.takeSnapshot() - val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs) + val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs, time) recoveredMapping.truncateAndReload(logStartOffset = 0L, logEndOffset = 1L, time.milliseconds) val lastEntry = recoveredMapping.lastEntry(producerId) @@ -542,7 +542,7 @@ class ProducerStateManagerTest { append(stateManager, producerId, epoch, 1, 1L, 1) stateManager.takeSnapshot() - val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs) + val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs, time) recoveredMapping.truncateAndReload(0L, 1L, 70000) // entry added after recovery. The pid should be expired now, and would not exist in the pid mapping. Hence @@ -561,7 +561,7 @@ class ProducerStateManagerTest { append(stateManager, producerId, epoch, 1, 1L, 1) stateManager.takeSnapshot() - val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs) + val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs, time) recoveredMapping.truncateAndReload(0L, 1L, 70000) val sequence = 2 @@ -769,7 +769,7 @@ class ProducerStateManagerTest { @Test def testSequenceNotValidatedForGroupMetadataTopic(): Unit = { val partition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, 0) - val stateManager = new ProducerStateManager(partition, logDir, maxPidExpirationMs) + val stateManager = new ProducerStateManager(partition, logDir, maxPidExpirationMs, time) val epoch = 0.toShort append(stateManager, producerId, epoch, RecordBatch.NO_SEQUENCE, offset = 99, @@ -818,7 +818,7 @@ class ProducerStateManagerTest { appendEndTxnMarker(stateManager, producerId, producerEpoch, ControlRecordType.COMMIT, offset = 100, coordinatorEpoch = 1) stateManager.takeSnapshot() - val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs) + val recoveredMapping = new ProducerStateManager(partition, logDir, maxPidExpirationMs, time) recoveredMapping.truncateAndReload(0L, 2L, 70000) // append from old coordinator should be rejected @@ -922,7 +922,7 @@ class ProducerStateManagerTest { } // Ensure that the truncated snapshot is deleted and producer state is loaded from the previous snapshot - val reloadedStateManager = new ProducerStateManager(partition, logDir, maxPidExpirationMs) + val reloadedStateManager = new ProducerStateManager(partition, logDir, maxPidExpirationMs, time) reloadedStateManager.truncateAndReload(0L, 20L, time.milliseconds()) assertFalse(snapshotToTruncate.exists()) diff --git a/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala b/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala index b0fa2b36b940f..418ee31e3ae03 100644 --- a/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala @@ -274,7 +274,8 @@ class ControllerApisTest { val request = buildRequest(brokerRegistrationRequest) val capturedResponse: ArgumentCaptor[AbstractResponse] = ArgumentCaptor.forClass(classOf[AbstractResponse]) - createControllerApis(Some(createDenyAllAuthorizer()), mock(classOf[Controller])).handle(request) + createControllerApis(Some(createDenyAllAuthorizer()), mock(classOf[Controller])).handle(request, + RequestLocal.withThreadConfinedCaching) verify(requestChannel).sendResponse( ArgumentMatchers.eq(request), capturedResponse.capture(), diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index bd5de0838b720..46f4c1bd51e64 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -313,7 +313,7 @@ class KafkaApisTest { EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, authorizer, adminManager, controller) - createKafkaApis(authorizer = Some(authorizer), enableForwarding = true).handle(request) + createKafkaApis(authorizer = Some(authorizer), enableForwarding = true).handle(request, RequestLocal.withThreadConfinedCaching) assertEquals(Some(request), capturedRequest.getValue.envelope) val innerResponse = capturedResponse.getValue.asInstanceOf[AlterConfigsResponse] @@ -339,7 +339,7 @@ class KafkaApisTest { val capturedResponse = expectNoThrottling(request) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, controller) - createKafkaApis(enableForwarding = true).handle(request) + createKafkaApis(enableForwarding = true).handle(request, RequestLocal.withThreadConfinedCaching) val response = capturedResponse.getValue.asInstanceOf[EnvelopeResponse] assertEquals(Errors.INVALID_REQUEST, response.error()) @@ -407,7 +407,7 @@ class KafkaApisTest { EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, authorizer, adminManager, controller) - createKafkaApis(authorizer = Some(authorizer), enableForwarding = true).handle(request) + createKafkaApis(authorizer = Some(authorizer), enableForwarding = true).handle(request, RequestLocal.withThreadConfinedCaching) if (!shouldCloseConnection) { val response = capturedResponse.getValue.asInstanceOf[EnvelopeResponse] @@ -482,7 +482,7 @@ class KafkaApisTest { EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, controller, forwardingManager) - createKafkaApis(enableForwarding = true).handle(request) + createKafkaApis(enableForwarding = true).handle(request, RequestLocal.withThreadConfinedCaching) EasyMock.verify(controller, forwardingManager) } @@ -1078,7 +1078,7 @@ class KafkaApisTest { val request = buildRequest(offsetCommitRequest) val capturedResponse = expectNoThrottling(request) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel) - createKafkaApis().handleOffsetCommitRequest(request) + createKafkaApis().handleOffsetCommitRequest(request, RequestLocal.withThreadConfinedCaching) val response = capturedResponse.getValue.asInstanceOf[OffsetCommitResponse] assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, @@ -1110,7 +1110,7 @@ class KafkaApisTest { val capturedResponse = expectNoThrottling(request) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel) - createKafkaApis().handleTxnOffsetCommitRequest(request) + createKafkaApis().handleTxnOffsetCommitRequest(request, RequestLocal.withThreadConfinedCaching) val response = capturedResponse.getValue.asInstanceOf[TxnOffsetCommitResponse] assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, response.errors().get(invalidTopicPartition)) @@ -1147,6 +1147,7 @@ class KafkaApisTest { ).build(version.toShort) val request = buildRequest(offsetCommitRequest) + val requestLocal = RequestLocal.withThreadConfinedCaching EasyMock.expect(groupCoordinator.handleTxnCommitOffsets( EasyMock.eq(groupId), EasyMock.eq(producerId), @@ -1155,7 +1156,8 @@ class KafkaApisTest { EasyMock.eq(Option.empty), EasyMock.anyInt(), EasyMock.anyObject(), - EasyMock.capture(responseCallback) + EasyMock.capture(responseCallback), + EasyMock.eq(requestLocal) )).andAnswer( () => responseCallback.getValue.apply(Map(topicPartition -> Errors.COORDINATOR_LOAD_IN_PROGRESS))) @@ -1167,7 +1169,7 @@ class KafkaApisTest { EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, groupCoordinator) - createKafkaApis().handleTxnOffsetCommitRequest(request) + createKafkaApis().handleTxnOffsetCommitRequest(request, requestLocal) val response = capturedResponse.getValue.asInstanceOf[TxnOffsetCommitResponse] @@ -1219,11 +1221,13 @@ class KafkaApisTest { else Option(new ProducerIdAndEpoch(producerId, epoch)) + val requestLocal = RequestLocal.withThreadConfinedCaching EasyMock.expect(txnCoordinator.handleInitProducerId( EasyMock.eq(transactionalId), EasyMock.eq(txnTimeoutMs), EasyMock.eq(expectedProducerIdAndEpoch), - EasyMock.capture(responseCallback) + EasyMock.capture(responseCallback), + EasyMock.eq(requestLocal) )).andAnswer( () => responseCallback.getValue.apply(InitProducerIdResult(producerId, epoch, Errors.PRODUCER_FENCED))) @@ -1235,7 +1239,7 @@ class KafkaApisTest { EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) - createKafkaApis().handleInitProducerIdRequest(request) + createKafkaApis().handleInitProducerIdRequest(request, requestLocal) val response = capturedResponse.getValue.asInstanceOf[InitProducerIdResponse] @@ -1278,12 +1282,14 @@ class KafkaApisTest { EasyMock.eq(groupId) )).andReturn(partition) + val requestLocal = RequestLocal.withThreadConfinedCaching EasyMock.expect(txnCoordinator.handleAddPartitionsToTransaction( EasyMock.eq(transactionalId), EasyMock.eq(producerId), EasyMock.eq(epoch), EasyMock.eq(Set(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partition))), - EasyMock.capture(responseCallback) + EasyMock.capture(responseCallback), + EasyMock.eq(requestLocal) )).andAnswer( () => responseCallback.getValue.apply(Errors.PRODUCER_FENCED)) @@ -1295,7 +1301,7 @@ class KafkaApisTest { EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator, groupCoordinator) - createKafkaApis().handleAddOffsetsToTxnRequest(request) + createKafkaApis().handleAddOffsetsToTxnRequest(request, requestLocal) val response = capturedResponse.getValue.asInstanceOf[AddOffsetsToTxnResponse] @@ -1334,13 +1340,14 @@ class KafkaApisTest { ).build(version.toShort) val request = buildRequest(addPartitionsToTxnRequest) + val requestLocal = RequestLocal.withThreadConfinedCaching EasyMock.expect(txnCoordinator.handleAddPartitionsToTransaction( EasyMock.eq(transactionalId), EasyMock.eq(producerId), EasyMock.eq(epoch), EasyMock.eq(Set(topicPartition)), - - EasyMock.capture(responseCallback) + EasyMock.capture(responseCallback), + EasyMock.eq(requestLocal) )).andAnswer( () => responseCallback.getValue.apply(Errors.PRODUCER_FENCED)) @@ -1352,7 +1359,7 @@ class KafkaApisTest { EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) - createKafkaApis().handleAddPartitionToTxnRequest(request) + createKafkaApis().handleAddPartitionToTxnRequest(request, requestLocal) val response = capturedResponse.getValue.asInstanceOf[AddPartitionsToTxnResponse] @@ -1388,12 +1395,14 @@ class KafkaApisTest { ).build(version.toShort) val request = buildRequest(endTxnRequest) + val requestLocal = RequestLocal.withThreadConfinedCaching EasyMock.expect(txnCoordinator.handleEndTransaction( EasyMock.eq(transactionalId), EasyMock.eq(producerId), EasyMock.eq(epoch), EasyMock.eq(TransactionResult.COMMIT), - EasyMock.capture(responseCallback) + EasyMock.capture(responseCallback), + EasyMock.eq(requestLocal) )).andAnswer( () => responseCallback.getValue.apply(Errors.PRODUCER_FENCED)) @@ -1404,7 +1413,7 @@ class KafkaApisTest { )) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) - createKafkaApis().handleEndTxnRequest(request) + createKafkaApis().handleEndTxnRequest(request, requestLocal) val response = capturedResponse.getValue.asInstanceOf[EndTxnResponse] @@ -1449,6 +1458,7 @@ class KafkaApisTest { EasyMock.anyObject(), EasyMock.capture(responseCallback), EasyMock.anyObject(), + EasyMock.anyObject(), EasyMock.anyObject()) ).andAnswer(() => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.INVALID_PRODUCER_EPOCH)))) @@ -1458,7 +1468,7 @@ class KafkaApisTest { EasyMock.replay(replicaManager, clientQuotaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) - createKafkaApis().handleProduceRequest(request) + createKafkaApis().handleProduceRequest(request, RequestLocal.withThreadConfinedCaching) val response = capturedResponse.getValue.asInstanceOf[ProduceResponse] @@ -1486,7 +1496,7 @@ class KafkaApisTest { val capturedResponse = expectNoThrottling(request) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel) - createKafkaApis().handleAddPartitionToTxnRequest(request) + createKafkaApis().handleAddPartitionToTxnRequest(request, RequestLocal.withThreadConfinedCaching) val response = capturedResponse.getValue.asInstanceOf[AddPartitionsToTxnResponse] assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, response.errors().get(invalidTopicPartition)) @@ -1498,27 +1508,32 @@ class KafkaApisTest { @Test def shouldThrowUnsupportedVersionExceptionOnHandleAddOffsetToTxnRequestWhenInterBrokerProtocolNotSupported(): Unit = { - assertThrows(classOf[UnsupportedVersionException], () => createKafkaApis(KAFKA_0_10_2_IV0).handleAddOffsetsToTxnRequest(null)) + assertThrows(classOf[UnsupportedVersionException], + () => createKafkaApis(KAFKA_0_10_2_IV0).handleAddOffsetsToTxnRequest(null, RequestLocal.withThreadConfinedCaching)) } @Test def shouldThrowUnsupportedVersionExceptionOnHandleAddPartitionsToTxnRequestWhenInterBrokerProtocolNotSupported(): Unit = { - assertThrows(classOf[UnsupportedVersionException], () => createKafkaApis(KAFKA_0_10_2_IV0).handleAddPartitionToTxnRequest(null)) + assertThrows(classOf[UnsupportedVersionException], + () => createKafkaApis(KAFKA_0_10_2_IV0).handleAddPartitionToTxnRequest(null, RequestLocal.withThreadConfinedCaching)) } @Test def shouldThrowUnsupportedVersionExceptionOnHandleTxnOffsetCommitRequestWhenInterBrokerProtocolNotSupported(): Unit = { - assertThrows(classOf[UnsupportedVersionException], () => createKafkaApis(KAFKA_0_10_2_IV0).handleAddPartitionToTxnRequest(null)) + assertThrows(classOf[UnsupportedVersionException], + () => createKafkaApis(KAFKA_0_10_2_IV0).handleAddPartitionToTxnRequest(null, RequestLocal.withThreadConfinedCaching)) } @Test def shouldThrowUnsupportedVersionExceptionOnHandleEndTxnRequestWhenInterBrokerProtocolNotSupported(): Unit = { - assertThrows(classOf[UnsupportedVersionException], () => createKafkaApis(KAFKA_0_10_2_IV0).handleEndTxnRequest(null)) + assertThrows(classOf[UnsupportedVersionException], + () => createKafkaApis(KAFKA_0_10_2_IV0).handleEndTxnRequest(null, RequestLocal.withThreadConfinedCaching)) } @Test def shouldThrowUnsupportedVersionExceptionOnHandleWriteTxnMarkersRequestWhenInterBrokerProtocolNotSupported(): Unit = { - assertThrows(classOf[UnsupportedVersionException], () => createKafkaApis(KAFKA_0_10_2_IV0).handleWriteTxnMarkersRequest(null)) + assertThrows(classOf[UnsupportedVersionException], + () => createKafkaApis(KAFKA_0_10_2_IV0).handleWriteTxnMarkersRequest(null, RequestLocal.withThreadConfinedCaching)) } @Test @@ -1537,7 +1552,7 @@ class KafkaApisTest { )) EasyMock.replay(replicaManager, replicaQuotaManager, requestChannel) - createKafkaApis().handleWriteTxnMarkersRequest(request) + createKafkaApis().handleWriteTxnMarkersRequest(request, RequestLocal.withThreadConfinedCaching) val markersResponse = capturedResponse.getValue.asInstanceOf[WriteTxnMarkersResponse] assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) @@ -1559,7 +1574,7 @@ class KafkaApisTest { )) EasyMock.replay(replicaManager, replicaQuotaManager, requestChannel) - createKafkaApis().handleWriteTxnMarkersRequest(request) + createKafkaApis().handleWriteTxnMarkersRequest(request, RequestLocal.withThreadConfinedCaching) val markersResponse = capturedResponse.getValue.asInstanceOf[WriteTxnMarkersResponse] assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) @@ -1580,6 +1595,7 @@ class KafkaApisTest { EasyMock.expect(replicaManager.getMagic(tp2)) .andReturn(Some(RecordBatch.MAGIC_VALUE_V2)) + val requestLocal = RequestLocal.withThreadConfinedCaching EasyMock.expect(replicaManager.appendRecords(EasyMock.anyLong(), EasyMock.anyShort(), EasyMock.eq(true), @@ -1587,7 +1603,8 @@ class KafkaApisTest { EasyMock.anyObject(), EasyMock.capture(responseCallback), EasyMock.anyObject(), - EasyMock.anyObject()) + EasyMock.anyObject(), + EasyMock.eq(requestLocal)) ).andAnswer(() => responseCallback.getValue.apply(Map(tp2 -> new PartitionResponse(Errors.NONE)))) EasyMock.expect(requestChannel.sendResponse( @@ -1597,7 +1614,7 @@ class KafkaApisTest { )) EasyMock.replay(replicaManager, replicaQuotaManager, requestChannel) - createKafkaApis().handleWriteTxnMarkersRequest(request) + createKafkaApis().handleWriteTxnMarkersRequest(request, requestLocal) val markersResponse = capturedResponse.getValue.asInstanceOf[WriteTxnMarkersResponse] assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) @@ -1719,6 +1736,7 @@ class KafkaApisTest { EasyMock.expect(replicaManager.getMagic(tp2)) .andReturn(Some(RecordBatch.MAGIC_VALUE_V2)) + val requestLocal = RequestLocal.withThreadConfinedCaching EasyMock.expect(replicaManager.appendRecords(EasyMock.anyLong(), EasyMock.anyShort(), EasyMock.eq(true), @@ -1726,7 +1744,8 @@ class KafkaApisTest { EasyMock.anyObject(), EasyMock.capture(responseCallback), EasyMock.anyObject(), - EasyMock.anyObject()) + EasyMock.anyObject(), + EasyMock.eq(requestLocal)) ).andAnswer(() => responseCallback.getValue.apply(Map(tp2 -> new PartitionResponse(Errors.NONE)))) EasyMock.expect(requestChannel.sendResponse( @@ -1736,7 +1755,7 @@ class KafkaApisTest { )) EasyMock.replay(replicaManager, replicaQuotaManager, requestChannel) - createKafkaApis().handleWriteTxnMarkersRequest(request) + createKafkaApis().handleWriteTxnMarkersRequest(request, requestLocal) val markersResponse = capturedResponse.getValue.asInstanceOf[WriteTxnMarkersResponse] assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) @@ -1750,6 +1769,7 @@ class KafkaApisTest { EasyMock.expect(replicaManager.getMagic(topicPartition)) .andReturn(Some(RecordBatch.MAGIC_VALUE_V2)) + val requestLocal = RequestLocal.withThreadConfinedCaching EasyMock.expect(replicaManager.appendRecords(EasyMock.anyLong(), EasyMock.anyShort(), EasyMock.eq(true), @@ -1757,11 +1777,12 @@ class KafkaApisTest { EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), - EasyMock.anyObject())) + EasyMock.anyObject(), + EasyMock.eq(requestLocal))) EasyMock.replay(replicaManager) - createKafkaApis().handleWriteTxnMarkersRequest(request) + createKafkaApis().handleWriteTxnMarkersRequest(request, requestLocal) EasyMock.verify(replicaManager) } @@ -1857,6 +1878,7 @@ class KafkaApisTest { ).build() val request = buildRequest(offsetDeleteRequest) + val requestLocal = RequestLocal.withThreadConfinedCaching val capturedResponse = expectNoThrottling(request) EasyMock.expect(groupCoordinator.handleDeleteOffsets( EasyMock.eq(group), @@ -1865,7 +1887,8 @@ class KafkaApisTest { new TopicPartition("topic-1", 1), new TopicPartition("topic-2", 0), new TopicPartition("topic-2", 1) - )) + )), + EasyMock.eq(requestLocal) )).andReturn((Errors.NONE, Map( new TopicPartition("topic-1", 0) -> Errors.NONE, new TopicPartition("topic-1", 1) -> Errors.NONE, @@ -1875,7 +1898,7 @@ class KafkaApisTest { EasyMock.replay(groupCoordinator, replicaManager, clientRequestQuotaManager, requestChannel) - createKafkaApis().handleOffsetDeleteRequest(request) + createKafkaApis().handleOffsetDeleteRequest(request, requestLocal) val response = capturedResponse.getValue.asInstanceOf[OffsetDeleteResponse] @@ -1912,11 +1935,12 @@ class KafkaApisTest { val request = buildRequest(offsetDeleteRequest) val capturedResponse = expectNoThrottling(request) - EasyMock.expect(groupCoordinator.handleDeleteOffsets(EasyMock.eq(group), EasyMock.eq(Seq.empty))) - .andReturn((Errors.NONE, Map.empty)) + val requestLocal = RequestLocal.withThreadConfinedCaching + EasyMock.expect(groupCoordinator.handleDeleteOffsets(EasyMock.eq(group), EasyMock.eq(Seq.empty), + EasyMock.eq(requestLocal))).andReturn((Errors.NONE, Map.empty)) EasyMock.replay(groupCoordinator, replicaManager, clientRequestQuotaManager, requestChannel) - createKafkaApis().handleOffsetDeleteRequest(request) + createKafkaApis().handleOffsetDeleteRequest(request, requestLocal) val response = capturedResponse.getValue.asInstanceOf[OffsetDeleteResponse] @@ -1941,11 +1965,12 @@ class KafkaApisTest { val request = buildRequest(offsetDeleteRequest) val capturedResponse = expectNoThrottling(request) - EasyMock.expect(groupCoordinator.handleDeleteOffsets(EasyMock.eq(group), EasyMock.eq(Seq.empty))) - .andReturn((Errors.GROUP_ID_NOT_FOUND, Map.empty)) + val requestLocal = RequestLocal.withThreadConfinedCaching + EasyMock.expect(groupCoordinator.handleDeleteOffsets(EasyMock.eq(group), EasyMock.eq(Seq.empty), + EasyMock.eq(requestLocal))).andReturn((Errors.GROUP_ID_NOT_FOUND, Map.empty)) EasyMock.replay(groupCoordinator, replicaManager, clientRequestQuotaManager, requestChannel) - createKafkaApis().handleOffsetDeleteRequest(request) + createKafkaApis().handleOffsetDeleteRequest(request, requestLocal) val response = capturedResponse.getValue.asInstanceOf[OffsetDeleteResponse] @@ -2174,6 +2199,7 @@ class KafkaApisTest { EasyMock.eq(sessionTimeoutMs), EasyMock.eq(protocolType), EasyMock.capture(capturedProtocols), + anyObject(), anyObject() )) @@ -2193,7 +2219,8 @@ class KafkaApisTest { .setName(name).setMetadata(protocol) }.iterator.asJava)) ).build() - )) + ), + RequestLocal.withThreadConfinedCaching) EasyMock.verify(groupCoordinator) @@ -2234,7 +2261,8 @@ class KafkaApisTest { EasyMock.eq(sessionTimeoutMs), EasyMock.eq(protocolType), EasyMock.eq(List.empty), - EasyMock.capture(capturedCallback) + EasyMock.capture(capturedCallback), + EasyMock.anyObject() )) val joinGroupRequest = new JoinGroupRequest.Builder( @@ -2250,7 +2278,7 @@ class KafkaApisTest { val capturedResponse = expectNoThrottling(requestChannelRequest) EasyMock.replay(groupCoordinator, clientRequestQuotaManager, requestChannel, replicaManager) - createKafkaApis().handleJoinGroupRequest(requestChannelRequest) + createKafkaApis().handleJoinGroupRequest(requestChannelRequest, RequestLocal.withThreadConfinedCaching) EasyMock.verify(groupCoordinator) @@ -2304,7 +2332,8 @@ class KafkaApisTest { EasyMock.eq(sessionTimeoutMs), EasyMock.eq(protocolType), EasyMock.eq(List.empty), - EasyMock.capture(capturedCallback) + EasyMock.capture(capturedCallback), + EasyMock.anyObject() )) val joinGroupRequest = new JoinGroupRequest.Builder( @@ -2320,7 +2349,7 @@ class KafkaApisTest { val capturedResponse = expectNoThrottling(requestChannelRequest) EasyMock.replay(groupCoordinator, clientRequestQuotaManager, requestChannel, replicaManager) - createKafkaApis().handleJoinGroupRequest(requestChannelRequest) + createKafkaApis().handleJoinGroupRequest(requestChannelRequest, RequestLocal.withThreadConfinedCaching) EasyMock.verify(groupCoordinator) @@ -2364,6 +2393,7 @@ class KafkaApisTest { val capturedCallback = EasyMock.newCapture[SyncGroupCallback]() + val requestLocal = RequestLocal.withThreadConfinedCaching EasyMock.expect(groupCoordinator.handleSyncGroup( EasyMock.eq(groupId), EasyMock.eq(0), @@ -2372,7 +2402,8 @@ class KafkaApisTest { EasyMock.eq(if (version >= 5) Some(protocolName) else None), EasyMock.eq(None), EasyMock.eq(Map.empty), - EasyMock.capture(capturedCallback) + EasyMock.capture(capturedCallback), + EasyMock.eq(requestLocal) )) val syncGroupRequest = new SyncGroupRequest.Builder( @@ -2388,7 +2419,7 @@ class KafkaApisTest { val capturedResponse = expectNoThrottling(requestChannelRequest) EasyMock.replay(groupCoordinator, clientRequestQuotaManager, requestChannel, replicaManager) - createKafkaApis().handleSyncGroupRequest(requestChannelRequest) + createKafkaApis().handleSyncGroupRequest(requestChannelRequest, requestLocal) EasyMock.verify(groupCoordinator) @@ -2425,6 +2456,7 @@ class KafkaApisTest { val capturedCallback = EasyMock.newCapture[SyncGroupCallback]() + val requestLocal = RequestLocal.withThreadConfinedCaching if (version < 5) { EasyMock.expect(groupCoordinator.handleSyncGroup( EasyMock.eq(groupId), @@ -2434,7 +2466,8 @@ class KafkaApisTest { EasyMock.eq(None), EasyMock.eq(None), EasyMock.eq(Map.empty), - EasyMock.capture(capturedCallback) + EasyMock.capture(capturedCallback), + EasyMock.eq(requestLocal) )) } @@ -2449,7 +2482,7 @@ class KafkaApisTest { val capturedResponse = expectNoThrottling(requestChannelRequest) EasyMock.replay(groupCoordinator, clientRequestQuotaManager, requestChannel, replicaManager) - createKafkaApis().handleSyncGroupRequest(requestChannelRequest) + createKafkaApis().handleSyncGroupRequest(requestChannelRequest, requestLocal) EasyMock.verify(groupCoordinator) @@ -2488,7 +2521,7 @@ class KafkaApisTest { val capturedResponse = expectNoThrottling(requestChannelRequest) EasyMock.replay(clientRequestQuotaManager, requestChannel) - createKafkaApis(KAFKA_2_2_IV1).handleJoinGroupRequest(requestChannelRequest) + createKafkaApis(KAFKA_2_2_IV1).handleJoinGroupRequest(requestChannelRequest, RequestLocal.withThreadConfinedCaching) val response = capturedResponse.getValue.asInstanceOf[JoinGroupResponse] assertEquals(Errors.UNSUPPORTED_VERSION, response.error()) @@ -2509,7 +2542,7 @@ class KafkaApisTest { val capturedResponse = expectNoThrottling(requestChannelRequest) EasyMock.replay(clientRequestQuotaManager, requestChannel) - createKafkaApis(KAFKA_2_2_IV1).handleSyncGroupRequest(requestChannelRequest) + createKafkaApis(KAFKA_2_2_IV1).handleSyncGroupRequest(requestChannelRequest, RequestLocal.withThreadConfinedCaching) val response = capturedResponse.getValue.asInstanceOf[SyncGroupResponse] assertEquals(Errors.UNSUPPORTED_VERSION, response.error) @@ -2561,7 +2594,7 @@ class KafkaApisTest { val capturedResponse = expectNoThrottling(requestChannelRequest) EasyMock.replay(clientRequestQuotaManager, requestChannel) - createKafkaApis(KAFKA_2_2_IV1).handleOffsetCommitRequest(requestChannelRequest) + createKafkaApis(KAFKA_2_2_IV1).handleOffsetCommitRequest(requestChannelRequest, RequestLocal.withThreadConfinedCaching) val expectedTopicErrors = Collections.singletonList( new OffsetCommitResponseData.OffsetCommitResponseTopic() @@ -2688,7 +2721,7 @@ class KafkaApisTest { replay(replicaManager, fetchManager, clientQuotaManager, requestChannel, replicaQuotaManager, partition) - createKafkaApis().handle(fetchFromFollower) + createKafkaApis().handle(fetchFromFollower, RequestLocal.withThreadConfinedCaching) if (isReassigning) assertEquals(records.sizeInBytes(), brokerTopicStats.allTopicsStats.reassignmentBytesOutPerSec.get.count()) @@ -2712,7 +2745,7 @@ class KafkaApisTest { val capturedResponse = expectNoThrottling(requestChannelRequest) EasyMock.replay(clientRequestQuotaManager, requestChannel) - createKafkaApis(KAFKA_2_2_IV1).handleInitProducerIdRequest(requestChannelRequest) + createKafkaApis(KAFKA_2_2_IV1).handleInitProducerIdRequest(requestChannelRequest, RequestLocal.withThreadConfinedCaching) val response = capturedResponse.getValue.asInstanceOf[InitProducerIdResponse] assertEquals(Errors.INVALID_REQUEST, response.error) @@ -2731,7 +2764,7 @@ class KafkaApisTest { val capturedResponse = expectNoThrottling(requestChannelRequest) EasyMock.replay(clientRequestQuotaManager, requestChannel) - createKafkaApis(KAFKA_2_2_IV1).handleInitProducerIdRequest(requestChannelRequest) + createKafkaApis(KAFKA_2_2_IV1).handleInitProducerIdRequest(requestChannelRequest, RequestLocal.withThreadConfinedCaching) val response = capturedResponse.getValue.asInstanceOf[InitProducerIdResponse] assertEquals(Errors.INVALID_REQUEST, response.error) @@ -2776,7 +2809,7 @@ class KafkaApisTest { )) EasyMock.replay(replicaManager, controller, requestChannel) - createKafkaApis().handleUpdateMetadataRequest(request) + createKafkaApis().handleUpdateMetadataRequest(request, RequestLocal.withThreadConfinedCaching) val updateMetadataResponse = capturedResponse.getValue.asInstanceOf[UpdateMetadataResponse] assertEquals(expectedError, updateMetadataResponse.error()) EasyMock.verify(replicaManager) @@ -3754,7 +3787,7 @@ class KafkaApisTest { @Test def testRaftShouldNeverHandleUpdateMetadataRequest(): Unit = { metadataCache = MetadataCache.raftMetadataCache(brokerId) - verifyShouldNeverHandle(createKafkaApis(raftSupport = true).handleUpdateMetadataRequest) + verifyShouldNeverHandle(createKafkaApis(raftSupport = true).handleUpdateMetadataRequest(_, RequestLocal.withThreadConfinedCaching)) } @Test @@ -3772,7 +3805,7 @@ class KafkaApisTest { @Test def testRaftShouldNeverHandleEnvelope(): Unit = { metadataCache = MetadataCache.raftMetadataCache(brokerId) - verifyShouldNeverHandle(createKafkaApis(raftSupport = true).handleEnvelope) + verifyShouldNeverHandle(createKafkaApis(raftSupport = true).handleEnvelope(_, RequestLocal.withThreadConfinedCaching)) } @Test diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 8081d0b994b32..65ab81f04230d 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -1075,7 +1075,6 @@ class ReplicaManagerTest { private def initializeLogAndTopicId(replicaManager: ReplicaManager, topicPartition: TopicPartition, topicId: Uuid): Unit = { val partition = replicaManager.createPartition(new TopicPartition(topic, 0)) val log = replicaManager.logManager.getOrCreateLog(topicPartition, false, false, Some(topicId)) - log.topicId = Some(topicId) partition.log = Some(log) } @@ -1486,8 +1485,8 @@ class ReplicaManagerTest { val tp = new TopicPartition(topic, topicPartition) val maxProducerIdExpirationMs = 30000 val segments = new LogSegments(tp) - val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, tp, mockLogDirFailureChannel, logConfig.messageFormatVersion.recordVersion) - val producerStateManager = new ProducerStateManager(tp, logDir, maxProducerIdExpirationMs) + val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, tp, mockLogDirFailureChannel, logConfig.messageFormatVersion.recordVersion, "") + val producerStateManager = new ProducerStateManager(tp, logDir, maxProducerIdExpirationMs, time) val offsets = LogLoader.load(LoadLogParams( logDir, tp, @@ -1517,7 +1516,7 @@ class ReplicaManagerTest { leaderEpochCache = leaderEpochCache, producerStateManager = producerStateManager, logDirFailureChannel = mockLogDirFailureChannel, - topicId = topicId, + _topicId = topicId, keepPartitionMetadataFile = true) { override def endOffsetForEpoch(leaderEpoch: Int): Option[OffsetAndEpoch] = { @@ -2195,7 +2194,7 @@ class ReplicaManagerTest { val batch = TestUtils.records(records = List( new SimpleRecord(10, "k1".getBytes, "v1".getBytes), new SimpleRecord(11, "k2".getBytes, "v2".getBytes))) - partition.appendRecordsToLeader(batch, AppendOrigin.Client, requiredAcks = 0) + partition.appendRecordsToLeader(batch, AppendOrigin.Client, requiredAcks = 0, RequestLocal.withThreadConfinedCaching) partition.log.get.updateHighWatermark(2L) partition.log.get.maybeIncrementLogStartOffset(1L, LeaderOffsetIncremented) replicaManager.logManager.checkpointLogRecoveryOffsets() diff --git a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataListenerTest.scala b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataListenerTest.scala index 89ba5f1d03328..f48a75d48dc45 100644 --- a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataListenerTest.scala +++ b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataListenerTest.scala @@ -94,7 +94,7 @@ class BrokerMetadataListenerTest { verify(groupCoordinator).handleDeletedPartitions(ArgumentMatchers.argThat[Seq[TopicPartition]] { partitions => partitions.toSet == partitionSet(topic, numPartitions) - }) + }, any()) val deleteImageCapture: ArgumentCaptor[MetadataImageBuilder] = ArgumentCaptor.forClass(classOf[MetadataImageBuilder]) diff --git a/core/src/test/scala/unit/kafka/utils/SchedulerTest.scala b/core/src/test/scala/unit/kafka/utils/SchedulerTest.scala index 876388d008012..b085ab0166513 100644 --- a/core/src/test/scala/unit/kafka/utils/SchedulerTest.scala +++ b/core/src/test/scala/unit/kafka/utils/SchedulerTest.scala @@ -123,8 +123,8 @@ class SchedulerTest { val topicPartition = Log.parseTopicPartitionName(logDir) val logDirFailureChannel = new LogDirFailureChannel(10) val segments = new LogSegments(topicPartition) - val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, logConfig.messageFormatVersion.recordVersion) - val producerStateManager = new ProducerStateManager(topicPartition, logDir, maxProducerIdExpirationMs) + val leaderEpochCache = Log.maybeCreateLeaderEpochCache(logDir, topicPartition, logDirFailureChannel, logConfig.messageFormatVersion.recordVersion, "") + val producerStateManager = new ProducerStateManager(topicPartition, logDir, maxProducerIdExpirationMs, mockTime) val offsets = LogLoader.load(LoadLogParams( logDir, topicPartition, @@ -143,7 +143,7 @@ class SchedulerTest { recoveryPoint = offsets.recoveryPoint, nextOffsetMetadata = offsets.nextOffsetMetadata, scheduler, brokerTopicStats, mockTime, LogManager.ProducerIdExpirationCheckIntervalMs, topicPartition, leaderEpochCache, producerStateManager, logDirFailureChannel, - topicId = None, keepPartitionMetadataFile = true) + _topicId = None, keepPartitionMetadataFile = true) assertTrue(scheduler.taskRunning(log.producerExpireCheck)) log.close() assertFalse(scheduler.taskRunning(log.producerExpireCheck)) diff --git a/docs/implementation.html b/docs/implementation.html index 78e7eb7405e7e..1476c2be52746 100644 --- a/docs/implementation.html +++ b/docs/implementation.html @@ -18,7 +18,7 @@